From ebe3c8c73d1ea7149309dc145c12af9779634563 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Tue, 20 Feb 2024 14:39:53 +0300 Subject: [PATCH 01/89] Add global _SESSION variable --- builtin-functions/_functions.txt | 2 ++ compiler/data/var-data.cpp | 2 +- compiler/vertex-util.cpp | 3 ++- runtime/interface.cpp | 3 ++- runtime/interface.h | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index 16077a9bc4..42875a11f1 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -123,6 +123,8 @@ global $_COOKIE; global $_REQUEST; /** @var mixed $_ENV */ global $_ENV; +/** @var mixed $_SESSION */ +global $_SESSION; /** @var mixed $argc */ global $argc; /** @var mixed $argv */ diff --git a/compiler/data/var-data.cpp b/compiler/data/var-data.cpp index 5aebcaaf27..148236076a 100644 --- a/compiler/data/var-data.cpp +++ b/compiler/data/var-data.cpp @@ -35,7 +35,7 @@ const ClassMemberInstanceField *VarData::as_class_instance_field() const { // TODO Dirty HACK, should be removed bool VarData::does_name_eq_any_builtin_global(const std::string &name) { static const std::unordered_set names = { - "_SERVER", "_GET", "_POST", "_FILES", "_COOKIE", "_REQUEST", "_ENV", "argc", "argv", + "_SERVER", "_GET", "_POST", "_FILES", "_COOKIE", "_REQUEST", "_ENV", "_SESSION", "argc", "argv", "MC", "MC_True", "config", "Durov", "FullMCTime", "KPHP_MC_WRITE_STAT_PROBABILITY", "d$PHP_SAPI"}; return names.find(name) != names.end(); diff --git a/compiler/vertex-util.cpp b/compiler/vertex-util.cpp index 7f6d1606c8..0a1717a16f 100644 --- a/compiler/vertex-util.cpp +++ b/compiler/vertex-util.cpp @@ -124,7 +124,8 @@ bool VertexUtil::is_superglobal(const std::string &s) { "_FILES", "_COOKIE", "_REQUEST", - "_ENV" + "_ENV", + "_SESSION" }; return vk::contains(names, s); } diff --git a/runtime/interface.cpp b/runtime/interface.cpp index 5a0432f01c..5686a2ffe2 100644 --- a/runtime/interface.cpp +++ b/runtime/interface.cpp @@ -900,6 +900,7 @@ mixed v$_FILES __attribute__ ((weak)); mixed v$_COOKIE __attribute__ ((weak)); mixed v$_REQUEST __attribute__ ((weak)); mixed v$_ENV __attribute__ ((weak)); +mixed v$_SESSION __attribute__ ((weak)); mixed v$argc __attribute__ ((weak)); mixed v$argv __attribute__ ((weak)); @@ -1467,9 +1468,9 @@ static void reset_superglobals() { hard_reset_var(v$_GET, array()); hard_reset_var(v$_POST, array()); hard_reset_var(v$_FILES, array()); - hard_reset_var(v$_COOKIE, array()); hard_reset_var(v$_REQUEST, array()); hard_reset_var(v$_ENV, array()); + hard_reset_var(v$_COOKIE, array()); dl::leave_critical_section(); } diff --git a/runtime/interface.h b/runtime/interface.h index ee84f3c8fe..dda1dcca3b 100644 --- a/runtime/interface.h +++ b/runtime/interface.h @@ -151,6 +151,7 @@ extern mixed v$_FILES; extern mixed v$_COOKIE; extern mixed v$_REQUEST; extern mixed v$_ENV; +extern mixed v$_SESSION; const int32_t UPLOAD_ERR_OK = 0; const int32_t UPLOAD_ERR_INI_SIZE = 1; From 6b2f3a6ab35ebde33b6c036a1954b1b8ca6c2b2f Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Wed, 20 Mar 2024 19:58:25 +0300 Subject: [PATCH 02/89] Define sessions 1. Define session class. 2. Define the basic structure of the basic functions. 3. Write the abstract logic of reading and writing session files with the stream and serialize functions. --- runtime/sessions.cpp | 270 +++++++++++++++++++++++++++++++++++++++++++ runtime/sessions.h | 54 +++++++++ 2 files changed, 324 insertions(+) create mode 100644 runtime/sessions.cpp create mode 100644 runtime/sessions.h diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp new file mode 100644 index 0000000000..6921ea8c57 --- /dev/null +++ b/runtime/sessions.cpp @@ -0,0 +1,270 @@ +#include "runtime/sessions.h" +#include "runtime/serialize-functions.h" +#include "runtime/interface.h" +// #include "runtime/files.h" +// #include "runtime/critical_section.h" + + +/* +TO-DO: +- generate_session_id() +- send_cookies() +- session_start(): check the id for dangerous symbols +- f$session_start(): + - check whether headers were sent or not + - parse argument const array options + - read_and_close + - session_flush() +- f$session_destroy() +*/ + +namespace sessions { + +static const SM = vk::singleton::get(); + +SM.tempdir_path = string(getenv("TMPDIR")); +SM.sessions_path = SM.tempdir_path + string("/sessions/"); +if (!f$file_exists(SM.sessions_path)) { + f$mkdir(SM.sessions_path); +} + +static bool generate_session_id() { + // pass +} + +bool SessionManager::session_write() { + if (!session_open()) { + return false; + } + f$file_put_contents(handler, string("")); // f$ftruncate(handler, 0); + f$rewind(handler); + { // additional local scope for temprory variables tdata and tres + string tdata = f$serialize(v$_SESSION); + mixed tres = f$file_put_contents(handler, tdata); + if (tres.is_bool() || static_cast(tres.as_int()) != tdata.size()) { + php_warning(); + return false; + } + } + return true; +} + +bool SessionManager::session_read() { + if ((handler.is_null()) || (!session_open())) { + php_warning(); + return false; + } + { // additional local scope for temprory variable tdata + Optional tdata; + data = f$file_get_contents(handler); + if (data.is_false() or data.is_null()) { + return false; + } + v$_SESSION = f$unserialize(data.val()); + } + return true; +} + +bool SessionManager::session_open() { + if (id.is_null()) { + return false; + } + if (!handler.is_null()) { + session_close(); + } + + if (session_path.is_null()) { + // setting the session_path is only here => can be unchecked if it exists + session_path = sessions_path + id.val() + string("/"); + if (!f$is_dir(session_path)) { + f$mkdir(session_path); + } + session_path += id.val() + string(".sess"); + } + + { // additional local scope for temprory variable thandler + mixed thandler = f$fopen(session_path, string("r+")); + handler = (!thandler.is_bool()) ? thandler : NULL; + } + if (handler.is_null()) { + return false; + } + return true; +} + +bool SessionManager::session_close() { + if (handler.is_null()) { + return true; + } + + if (!f$fclose(handler)) { + php_warning(); + return false; + } + handler = NULL; + return true; +} + +static bool session_abort() { + if (SM.get_session_status()) { + SM.session_close(); + SM.set_session_status(0); + return true; + } + return false; +} + +static bool session_flush() { + // pass +} + +static bool send_cookies() { + // 1. check whether headers were sent or not + // pass: can return false here + + // 2. check session_name for deprecated symbols, if it's user supplied + // pass: can return false here + + // 3. encode session_name, if it's user supplied + // pass: can return false here + + // 4. set specific headers in the cookies + f$setcookie(/* some data here */); + return true; +} + +static bool session_reset_id() { + if (SM.get_session_id().is_null()) { + php_warning("Cannot set session ID - session ID is not initialized"); + return false; + } + + if (SM.get_send_cookie()) { + send_cookies(); // TO-DO + SM.set_send_cookie(0); + } + + return true; +} + +static bool session_initialize() { + SM.set_session_status(1); + + SM.session_open(); + if (SM.get_handler().is_null()) { + session_abort(); + php_warning(); + return false; + } + + if (SM.get_session_id().is_null()) { + SM.generate_session_id(); // TO-DO + if (SM.get_session_id().is_null()) { + session_abort(); + php_warning("Failed to create session ID: %s (path: %s)", SM.session_name.c_str(), SM.get_session_path().val().c_str()); + return false; + } + SM.set_send_cookie(1); + } + + session_reset_id(); + + if (!SM.session_read()) { + SM.session_abort(); + php_warning("Failed to read session data: %s (path: %s)", SM.session_name.c_str(), SM.get_session_path().val().c_str()); + return false; + } + return true; +} + +static void session_start() { + // 1. check status of the session + if (SM.get_session_status()) { + if (SM.get_session_path().has_value()) { + php_warning("Ignoring session_start() because a session is already active (started from %s)", SM.get_session_path().val().c_str()); + } else { + php_warning("Ignoring session_start() because a session is already active"); + } + return false; + } + + SM.set_send_cookie(1); + + if (SM.get_session_name().is_null()) { + SM.set_session_name("PHPSESSID"); // or "KPHPSESSID" + } + + // 2. check id of session + if (!SM.get_session_id().is_null()) { + // 4.1 try to get an id of session via it's name from _COOKIE + mixed pid = (v$_COOKIE.has_key(SM.session_name)) ? v$_COOKIE.get_value(SM.session_name) : NULL; + if (!pid.is_string()) { + pid = NULL; + } else { + SM.set_send_cookie(0); + } + SM.set_session_id(pid); + } + + // 5. check the id for dangerous symbols + // pass + + // 6. try to initialize the session + if (!session_initialize()) { + SM.set_session_status(0); + SM.set_session_id(NULL); + return false; + } + return true; +} + +} // namespace sessions + +bool f$session_start(/*const array &options*/) { + // 1. check status of session + if (sessions::SM.get_session_status()) { + if (sessions::SM.get_session_path().has_value()) { + php_warning("Ignoring session_start() because a session is already active (started from %s)", sessions::SM.get_session_path().val().c_str()); + } else { + php_warning("Ignoring session_start() because a session is already active"); + } + return true; + } + + // 2. check whether headers were sent or not + // for this kphp must have the headers_sent() function + + // bool read_and_close = false; + + // 3. parse the options arg and set it + // if (!options.empty()) { + // for (auto it = options.begin(); it != options.end(); ++it) { + // if (it.get_key().to_string() == "read_and_close") { + // read_and_close = true; + // } else { + // // 3.1 get the value and modify the session header with the same key + // // store to the buffer data + // // pass + // } + // } + // } + + sessions::session_start(); + + // 4. check the status of the session + if (!sessions::MS.get_session_status()) { + // 4.1 clear cache of session variables + // pass + php_warning(); + return false; + } + + // 5. check for read_and_close + // if (read_and_close) { + // // session_flush() + // // pass + // } + + return true; +} + diff --git a/runtime/sessions.h b/runtime/sessions.h new file mode 100644 index 0000000000..fdf99fce1e --- /dev/null +++ b/runtime/sessions.h @@ -0,0 +1,54 @@ +#pragma once + +#include "runtime/kphp_core.h" +#include "runtime/streams.h" + +namespace sessions { + +class SessionManager : vk::not_copyable { +public: + bool session_open(); + bool session_close(); + bool session_read(); + bool session_write(); + bool session_delete(); + + const bool &set_session_status(const bool &status) const noexcept { return session_status = status; } + const bool &get_session_status() const noexcept { return session_status; } + const Optional &set_session_path(const bool &path) const noexcept { return session_path = path; } + const Optional &get_session_path() const noexcept { return session_path; } + const Optional &set_session_id(const string &sid) const noexcept { return id = sid; } + const Optional &get_session_id() const noexcept { return id; } + const Optional &set_send_cookie(const bool &val) const noexcept { return send_cookie = val; } + const Optional &get_send_cookie() const noexcept { return send_cookie; } + const Optional &set_session_name(const string &val) const noexcept { return session_name = val; } + const Optional &get_session_name() const noexcept { return session_name; } + + const Optional &set_handler(const Stream &stream) const noexcept {return handler = stream; } + const Optional &get_handler() const noexcept {return handler; } + +private: + SessionManager() = default; + + Optional id{NULL}; + bool session_status{0}; + bool send_cookie{0}; + Optional session_path{NULL}; + Optional session_name{NULL}; // by default PHPSESSID (key for searching in the cookies) + + const string tempdir_path; // path to the /tmp + const string sessions_path; // path to the dir with existing sessions + + Optional handler{NULL}; + + friend class vk::singleton; +}; + +} // namespace sessions + +bool f$session_start(); +bool f$session_abort(); +bool f$session_commit(); +bool f$session_write_close(); +int64_t f$session_status(); +Optional f$session_create_id(const string &prefix); From 362fbd1e23441bbd01931f9628d9852a75700eb1 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Wed, 27 Mar 2024 17:26:24 +0300 Subject: [PATCH 03/89] Add sessions to compilation target --- runtime/runtime.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 7bfc8d802f..cfa8b1b0f2 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -123,7 +123,8 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ vkext_stats.cpp ffi.cpp zlib.cpp - zstd.cpp) + zstd.cpp + sessions.cpp) set_source_files_properties( ${BASE_DIR}/server/php-engine.cpp From 14fa55935797ff4e324ff4760c60a4639ad2d9c3 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Wed, 27 Mar 2024 17:27:12 +0300 Subject: [PATCH 04/89] Stage non-working version of sessions --- builtin-functions/_functions.txt | 8 + runtime/interface.cpp | 2 + runtime/sessions.cpp | 358 ++++++++++++++++++++++--------- runtime/sessions.h | 69 ++++-- 4 files changed, 318 insertions(+), 119 deletions(-) diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index 42875a11f1..b4ee7d1fde 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -772,6 +772,14 @@ function mb_strtolower ($str ::: string, $encoding ::: string = "cp1251") ::: st function mb_strtoupper ($str ::: string, $encoding ::: string = "cp1251") ::: string; function mb_substr ($str ::: string, $start ::: int, $length ::: mixed = PHP_INT_MAX, $encoding ::: string = "cp1251") ::: string; +function session_start(?mixed $options = null) ::: bool; +function session_abort() ::: bool; +function session_commit() ::: bool; +function session_write_close() ::: bool; +function session_gc() ::: bool; +function session_status() ::: bool; +function session_destroy() ::: bool; + define('PHP_ROUND_HALF_UP', 123423141); define('PHP_ROUND_HALF_DOWN', 123423144); define('PHP_ROUND_HALF_EVEN', 123423145); diff --git a/runtime/interface.cpp b/runtime/interface.cpp index 5686a2ffe2..1c5e9cb465 100644 --- a/runtime/interface.cpp +++ b/runtime/interface.cpp @@ -56,6 +56,7 @@ #include "runtime/udp.h" #include "runtime/url.h" #include "runtime/zlib.h" +// #include "runtime/sessions.h" #include "server/curl-adaptor.h" #include "server/database-drivers/adaptor.h" #include "server/database-drivers/mysql/mysql.h" @@ -1471,6 +1472,7 @@ static void reset_superglobals() { hard_reset_var(v$_REQUEST, array()); hard_reset_var(v$_ENV, array()); hard_reset_var(v$_COOKIE, array()); + hard_reset_var(v$_SESSION, mixed(NULL)); // Is this correct? dl::leave_critical_section(); } diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index 6921ea8c57..70951f06db 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -1,48 +1,79 @@ +#include +#include + #include "runtime/sessions.h" #include "runtime/serialize-functions.h" #include "runtime/interface.h" -// #include "runtime/files.h" +#include "runtime/streams.h" +#include "runtime/url.h" +#include "runtime/files.h" +#include "runtime/misc.h" // #include "runtime/critical_section.h" - -/* -TO-DO: -- generate_session_id() -- send_cookies() -- session_start(): check the id for dangerous symbols -- f$session_start(): - - check whether headers were sent or not - - parse argument const array options - - read_and_close - - session_flush() -- f$session_destroy() -*/ - namespace sessions { -static const SM = vk::singleton::get(); +SessionManager::SessionManager() { + tempdir_path = string(getenv("TMPDIR")); + sessions_path = string(tempdir_path).append("sessions/"); + if (!f$file_exists(sessions_path)) { + f$mkdir(sessions_path); + } +} + +static SessionManager &SM = vk::singleton::get(); -SM.tempdir_path = string(getenv("TMPDIR")); -SM.sessions_path = SM.tempdir_path + string("/sessions/"); -if (!f$file_exists(SM.sessions_path)) { - f$mkdir(SM.sessions_path); +static bool session_valid_id(const string &id) { + if (id.empty()) { + return false; + } + bool result = true; + for (auto i = (string("sess_").size()); i < id.size(); ++i) { + if (!((id[i] >= 'a' && id[i] <= 'z') + || (id[i] >= 'A' && id[i] <= 'Z') + || (id[i] >= '0' && id[i] <= '9') + || (id[i] == ',') + || (id[i] == '-'))) { + result = false; + break; + } + } + return result; } static bool generate_session_id() { - // pass + if (!SM.get_session_status()) { + return false; + } + string id = f$uniqid(string("sess_")); + + if (!session_valid_id(id)) { + php_warning("Failed to create new ID\n"); + return false; + } + SM.set_session_id(id); + return true; +} + +void SessionManager::session_reset_vars() { + handler = NULL; + id = false; + session_status = send_cookie = 0; + cookie_secure = cookie_httponly = 0; + cookie_lifetime = 0; + session_path = session_name = cookie_domain = false; } bool SessionManager::session_write() { if (!session_open()) { return false; } - f$file_put_contents(handler, string("")); // f$ftruncate(handler, 0); + f$file_put_contents(handler.to_string(), string("")); // f$ftruncate(handler, 0); f$rewind(handler); { // additional local scope for temprory variables tdata and tres string tdata = f$serialize(v$_SESSION); - mixed tres = f$file_put_contents(handler, tdata); + mixed tres = f$file_put_contents(handler.to_string(), tdata); if (tres.is_bool() || static_cast(tres.as_int()) != tdata.size()) { - php_warning(); + // php_warning(); return false; } } @@ -51,39 +82,42 @@ bool SessionManager::session_write() { bool SessionManager::session_read() { if ((handler.is_null()) || (!session_open())) { - php_warning(); + // php_warning(); return false; } { // additional local scope for temprory variable tdata Optional tdata; - data = f$file_get_contents(handler); - if (data.is_false() or data.is_null()) { + tdata = f$file_get_contents(handler.to_string()); + if (tdata.is_false()) { return false; } - v$_SESSION = f$unserialize(data.val()); + v$_SESSION = f$unserialize(tdata.val()); // TO-DO: check } + fprintf(stdout, "successfully read the session data with id %s\n", SM.get_session_id().val().c_str()); return true; } bool SessionManager::session_open() { - if (id.is_null()) { + if (id.is_false()) { return false; } if (!handler.is_null()) { session_close(); } - if (session_path.is_null()) { - // setting the session_path is only here => can be unchecked if it exists - session_path = sessions_path + id.val() + string("/"); - if (!f$is_dir(session_path)) { - f$mkdir(session_path); - } - session_path += id.val() + string(".sess"); + if (session_path.is_false()) { + // setting the session_path is only here + session_path = sessions_path; + session_path.val().append(id.val()); + session_path.val().append(".sess"); } + if (!f$file_exists(session_path.val())) { + f$fclose(f$fopen(session_path.val(), string("w+"))); + } + { // additional local scope for temprory variable thandler - mixed thandler = f$fopen(session_path, string("r+")); + mixed thandler = f$fopen(session_path.val(), string("r+")); handler = (!thandler.is_bool()) ? thandler : NULL; } if (handler.is_null()) { @@ -92,16 +126,76 @@ bool SessionManager::session_open() { return true; } -bool SessionManager::session_close() { +static bool session_destroy(const string &filepath) { + + Optional ctime = f$filectime(filepath); + if (ctime.is_false()) { + php_warning("Failed to get metadata of file %s\n", filepath.c_str()); + } + + // Stream thandler = f$fopen(filepath, string("r")); + // if (!thandler) { + // php_warning("Failed to open file %s\n", filepath.c_str()); + // return false; + // } + + Optional sdata = f$file_get_contents(filepath); + if (sdata.is_false()) { + php_warning("Failed to read file %s\n", filepath.c_str()); + return false; + } + array tdata = f$unserialize(sdata.val()).to_array(); + int lifetime = 0; + if (tdata.has_key(string("gc_maxlifetime"))) { + lifetime = tdata.get_value(string("gc_maxlifetime")).to_int(); + } + if (ctime < lifetime) { + // f$fclose(thandler); + return false; + } + // f$fclose(thandler); + return f$unlink(filepath); +} + +static bool session_gc() { + Optional> sfiles = f$scandir(SM.get_sessions_path()); + if (sfiles.is_false()) { + php_warning("Failed to scan sessions directory %s\n", SM.get_sessions_path().c_str()); + return false; + } + fprintf(stdout, "result of scandir is:\n%s\n", f$serialize(sfiles.val()).c_str()); + auto nsfiles = sfiles.val().count(); + int n = 0; + for (auto sfile = sfiles.val().cbegin(); sfile != sfiles.val().cend(); ++sfile) { + string val = sfile.get_value().to_string(); + if (val == string(".") or val == string("..")) { + continue; + } + if (session_destroy(val)) { + ++n; + } + } + if (n != nsfiles) { + php_warning("Failed to delete all files in sessions dir %s\n", SM.get_sessions_path().c_str()); + return false; + } + return true; +} + +bool SessionManager::session_close(bool immediate) { if (handler.is_null()) { return true; } if (!f$fclose(handler)) { - php_warning(); + // php_warning(); return false; } - handler = NULL; + session_reset_vars(); + + if (immediate or rand() % 100 > 75) { + session_gc(); + } return true; } @@ -114,33 +208,50 @@ static bool session_abort() { return false; } -static bool session_flush() { - // pass +static bool session_flush(bool write = false) { + if (!SM.get_session_status()) { + return false; + } + bool result = true; + if (write && !SM.session_write()) { + php_warning("Failed to write session data %s\n", SM.get_session_path().val().c_str()); + result = false; + } + + SM.set_session_status(0); + SM.session_close(); + return result; } static bool send_cookies() { // 1. check whether headers were sent or not - // pass: can return false here + // pass: need headers_sent() // 2. check session_name for deprecated symbols, if it's user supplied - // pass: can return false here + if (strpbrk(SM.get_session_name().val().c_str(), "=,;.[ \t\r\n\013\014") != NULL) { + php_warning("session.name cannot contain any of the following '=,;.[ \\t\\r\\n\\013\\014'"); + return false; + } // 3. encode session_name, if it's user supplied - // pass: can return false here - - // 4. set specific headers in the cookies - f$setcookie(/* some data here */); + int expire = SM.get_session_lifetime(); + if (expire > 0) { + expire += std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + } + string domain = !SM.get_cookie_domain().is_false() ? SM.get_cookie_domain().val() : string(""); + f$setcookie(SM.get_session_name().val(), f$urlencode(SM.get_session_id().val()), expire, string("/"), domain, SM.get_cookie_secure(), SM.get_cookie_httponly()); + fprintf(stdout, "successfully sent cookies for id %s\n", SM.get_session_id().val().c_str()); return true; } static bool session_reset_id() { - if (SM.get_session_id().is_null()) { + if (SM.get_session_id().is_false()) { php_warning("Cannot set session ID - session ID is not initialized"); return false; } if (SM.get_send_cookie()) { - send_cookies(); // TO-DO + send_cookies(); SM.set_send_cookie(0); } @@ -150,37 +261,38 @@ static bool session_reset_id() { static bool session_initialize() { SM.set_session_status(1); - SM.session_open(); - if (SM.get_handler().is_null()) { - session_abort(); - php_warning(); - return false; - } - - if (SM.get_session_id().is_null()) { - SM.generate_session_id(); // TO-DO - if (SM.get_session_id().is_null()) { + if (SM.get_session_id().is_false()) { + generate_session_id(); + if (SM.get_session_id().is_false()) { session_abort(); - php_warning("Failed to create session ID: %s (path: %s)", SM.session_name.c_str(), SM.get_session_path().val().c_str()); + php_warning("Failed to create session ID: %s (path: %s)", SM.get_session_name().val().c_str(), SM.get_session_path().val().c_str()); return false; } SM.set_send_cookie(1); } + + SM.session_open(); + if (SM.get_handler().is_null()) { + session_abort(); + // php_warning(); + return false; + } session_reset_id(); if (!SM.session_read()) { - SM.session_abort(); - php_warning("Failed to read session data: %s (path: %s)", SM.session_name.c_str(), SM.get_session_path().val().c_str()); + session_abort(); + php_warning("Failed to read session data: %s (path: %s)\n", SM.get_session_name().val().c_str(), SM.get_session_path().val().c_str()); return false; } + return true; } -static void session_start() { +static bool session_start() { // 1. check status of the session if (SM.get_session_status()) { - if (SM.get_session_path().has_value()) { + if (SM.get_session_path().is_false()) { php_warning("Ignoring session_start() because a session is already active (started from %s)", SM.get_session_path().val().c_str()); } else { php_warning("Ignoring session_start() because a session is already active"); @@ -190,29 +302,34 @@ static void session_start() { SM.set_send_cookie(1); - if (SM.get_session_name().is_null()) { - SM.set_session_name("PHPSESSID"); // or "KPHPSESSID" + if (SM.get_session_name().is_false()) { + SM.set_session_name(string("PHPSESSID")); } + fprintf(stdout, "session name is: %s\n", SM.get_session_name().val().c_str()); // 2. check id of session - if (!SM.get_session_id().is_null()) { + if (SM.get_session_id().is_false()) { // 4.1 try to get an id of session via it's name from _COOKIE - mixed pid = (v$_COOKIE.has_key(SM.session_name)) ? v$_COOKIE.get_value(SM.session_name) : NULL; - if (!pid.is_string()) { - pid = NULL; - } else { + Optional pid{false}; + if (v$_COOKIE.as_array().has_key(SM.get_session_name().val())) { + pid = v$_COOKIE.get_value(SM.get_session_name().val()).to_string(); + fprintf(stdout, "found session id in cookies: %s\n", pid.val().c_str()); + } + if (!pid.is_false()) { SM.set_send_cookie(0); } SM.set_session_id(pid); } // 5. check the id for dangerous symbols - // pass + if (!SM.get_session_id().is_false() && strpbrk(SM.get_session_id().val().c_str(), "\r\n\t <>'\"\\")) { + SM.set_session_id(false); + } // 6. try to initialize the session if (!session_initialize()) { SM.set_session_status(0); - SM.set_session_id(NULL); + SM.set_session_id(false); return false; } return true; @@ -220,10 +337,10 @@ static void session_start() { } // namespace sessions -bool f$session_start(/*const array &options*/) { +bool f$session_start(const array &options) { // 1. check status of session if (sessions::SM.get_session_status()) { - if (sessions::SM.get_session_path().has_value()) { + if (sessions::SM.get_session_path().is_false()) { php_warning("Ignoring session_start() because a session is already active (started from %s)", sessions::SM.get_session_path().val().c_str()); } else { php_warning("Ignoring session_start() because a session is already active"); @@ -232,39 +349,86 @@ bool f$session_start(/*const array &options*/) { } // 2. check whether headers were sent or not - // for this kphp must have the headers_sent() function + // need headers_sent() + // pass - // bool read_and_close = false; + bool read_and_close = false; // 3. parse the options arg and set it - // if (!options.empty()) { - // for (auto it = options.begin(); it != options.end(); ++it) { - // if (it.get_key().to_string() == "read_and_close") { - // read_and_close = true; - // } else { - // // 3.1 get the value and modify the session header with the same key - // // store to the buffer data - // // pass - // } - // } - // } - + if (!options.empty()) { + for (auto it = options.begin(); it != options.end(); ++it) { + string tkey = it.get_key().to_string(); + if (tkey == string("read_and_close")) { + read_and_close = it.get_value().to_bool(); + } else if (tkey == string("name")) { + sessions::SM.set_session_name(it.get_value().to_string()); + } else if (tkey == string("gc_maxlifetime")) { + sessions::SM.set_session_lifetime(it.get_value().to_int()); + } else if (tkey == string("cookie_path")) { + sessions::SM.set_session_path(it.get_value().to_string()); + } else if (tkey == string("cookie_lifetime")) { + sessions::SM.set_cookie_lifetime(it.get_value().to_int()); + } else if (tkey == string("cookie_domain")) { + sessions::SM.set_cookie_domain(it.get_value().to_string()); + } else if (tkey == string("cookie_secure")) { + sessions::SM.set_cookie_secure(it.get_value().to_bool()); + } else if (tkey == string("cookie_httponly")) { + sessions::SM.set_cookie_httponly(it.get_value().to_bool()); + } + } + } + fprintf(stdout, "Before session_start()\n"); sessions::session_start(); + fprintf(stdout, "After session_start()\n"); // 4. check the status of the session - if (!sessions::MS.get_session_status()) { + if (!sessions::SM.get_session_status()) { // 4.1 clear cache of session variables - // pass - php_warning(); + v$_SESSION = NULL; + // php_warning(); return false; } // 5. check for read_and_close - // if (read_and_close) { - // // session_flush() - // // pass - // } + if (read_and_close) { + sessions::session_flush(); + } return true; } +bool f$session_destroy() { + if (!sessions::SM.get_session_status()) { + php_warning("Trying to destroy uninitialized session"); + return false; + } + + if (sessions::SM.get_session_path().has_value()) { + return sessions::session_destroy(sessions::SM.get_session_path().val()); + } + sessions::SM.session_close(); + return false; +} + +bool f$session_abort() { + if (!sessions::SM.get_session_status()) { + return false; + } + return sessions::session_abort(); +} + +int64_t f$session_status() { + return static_cast(sessions::SM.get_session_status()) + 1; +} + +bool f$session_write_close() { + if (!sessions::SM.get_session_status()) { + return false; + } + return sessions::session_flush(1); +} + +bool $session_commit() { + return f$session_write_close(); +} + diff --git a/runtime/sessions.h b/runtime/sessions.h index fdf99fce1e..e66a5d0c5e 100644 --- a/runtime/sessions.h +++ b/runtime/sessions.h @@ -1,53 +1,78 @@ #pragma once #include "runtime/kphp_core.h" -#include "runtime/streams.h" +#include "common/mixin/not_copyable.h" +#include "common/smart_ptrs/singleton.h" namespace sessions { +using Stream = mixed; + class SessionManager : vk::not_copyable { public: bool session_open(); - bool session_close(); + bool session_close(bool immediate = false); bool session_read(); bool session_write(); bool session_delete(); - const bool &set_session_status(const bool &status) const noexcept { return session_status = status; } - const bool &get_session_status() const noexcept { return session_status; } - const Optional &set_session_path(const bool &path) const noexcept { return session_path = path; } - const Optional &get_session_path() const noexcept { return session_path; } - const Optional &set_session_id(const string &sid) const noexcept { return id = sid; } - const Optional &get_session_id() const noexcept { return id; } - const Optional &set_send_cookie(const bool &val) const noexcept { return send_cookie = val; } - const Optional &get_send_cookie() const noexcept { return send_cookie; } - const Optional &set_session_name(const string &val) const noexcept { return session_name = val; } - const Optional &get_session_name() const noexcept { return session_name; } + void session_reset_vars(); + + void set_session_status(const bool &status) noexcept { session_status = status; } + void set_session_path(const string &path) noexcept { session_path = path; } + void set_session_id(const Optional &sid) noexcept { id = sid; } + void set_send_cookie(const bool &val) noexcept { send_cookie = val; } + void set_session_name(const string &val) noexcept { session_name = val; } + void set_session_lifetime(const int &val) noexcept { session_lifetime = val; } + void set_handler(const Stream &stream) noexcept { handler = stream; } - const Optional &set_handler(const Stream &stream) const noexcept {return handler = stream; } - const Optional &get_handler() const noexcept {return handler; } + void set_cookie_domain(const string &val) noexcept { cookie_domain = val; } + void set_cookie_lifetime(const int &val) noexcept { cookie_lifetime = val; } + void set_cookie_secure(const bool &val) noexcept { cookie_secure = val; } + void set_cookie_httponly(const bool &val) noexcept { cookie_httponly = val; } + + bool get_session_status() noexcept { return session_status; } + Optional get_session_path() noexcept { return session_path; } + Optional get_session_id() noexcept { return id; } + bool get_send_cookie() noexcept { return send_cookie; } + Optional get_session_name() noexcept { return session_name; } + int get_session_lifetime() noexcept { return session_lifetime; } + Stream get_handler() noexcept {return handler; } + + string get_sessions_path() noexcept { return sessions_path; } + Optional get_cookie_domain() noexcept { return cookie_domain; } + int get_cookie_lifetime() noexcept { return cookie_lifetime; } + bool get_cookie_secure() noexcept { return cookie_secure; } + bool get_cookie_httponly() noexcept { return cookie_httponly; } private: - SessionManager() = default; + SessionManager(); - Optional id{NULL}; + Optional id{false}; bool session_status{0}; bool send_cookie{0}; - Optional session_path{NULL}; - Optional session_name{NULL}; // by default PHPSESSID (key for searching in the cookies) + Optional session_path{false}; + Optional session_name{false}; // by default PHPSESSID (key for searching in the cookies) + int session_lifetime{1440}; + + int cookie_lifetime{0}; + Optional cookie_domain{false}; + bool cookie_secure{0}; + bool cookie_httponly{0}; - const string tempdir_path; // path to the /tmp - const string sessions_path; // path to the dir with existing sessions + string tempdir_path; // path to the /tmp + string sessions_path; // path to the dir with existing sessions - Optional handler{NULL}; + Stream handler{NULL}; friend class vk::singleton; }; } // namespace sessions -bool f$session_start(); +bool f$session_start(const array &options = array()); bool f$session_abort(); +bool f$session_destroy(); bool f$session_commit(); bool f$session_write_close(); int64_t f$session_status(); From 0774419dc688d000b0f67b2c5608aaf8a7283a3b Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Mon, 1 Apr 2024 12:23:52 +0300 Subject: [PATCH 05/89] Add new global array for session handler --- compiler/data/var-data.cpp | 2 +- compiler/vertex-util.cpp | 1 + runtime/interface.cpp | 5 +++-- runtime/interface.h | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/data/var-data.cpp b/compiler/data/var-data.cpp index 148236076a..db9553e4e7 100644 --- a/compiler/data/var-data.cpp +++ b/compiler/data/var-data.cpp @@ -35,7 +35,7 @@ const ClassMemberInstanceField *VarData::as_class_instance_field() const { // TODO Dirty HACK, should be removed bool VarData::does_name_eq_any_builtin_global(const std::string &name) { static const std::unordered_set names = { - "_SERVER", "_GET", "_POST", "_FILES", "_COOKIE", "_REQUEST", "_ENV", "_SESSION", "argc", "argv", + "_SERVER", "_GET", "_POST", "_FILES", "_COOKIE", "_REQUEST", "_ENV", "_SESSION", "_KPHPSESSARR", "argc", "argv", "MC", "MC_True", "config", "Durov", "FullMCTime", "KPHP_MC_WRITE_STAT_PROBABILITY", "d$PHP_SAPI"}; return names.find(name) != names.end(); diff --git a/compiler/vertex-util.cpp b/compiler/vertex-util.cpp index 0a1717a16f..595deea0bd 100644 --- a/compiler/vertex-util.cpp +++ b/compiler/vertex-util.cpp @@ -126,6 +126,7 @@ bool VertexUtil::is_superglobal(const std::string &s) { "_REQUEST", "_ENV", "_SESSION" + "_KPHPSESSARR" }; return vk::contains(names, s); } diff --git a/runtime/interface.cpp b/runtime/interface.cpp index 1c5e9cb465..c6dc2dacc5 100644 --- a/runtime/interface.cpp +++ b/runtime/interface.cpp @@ -56,7 +56,6 @@ #include "runtime/udp.h" #include "runtime/url.h" #include "runtime/zlib.h" -// #include "runtime/sessions.h" #include "server/curl-adaptor.h" #include "server/database-drivers/adaptor.h" #include "server/database-drivers/mysql/mysql.h" @@ -902,6 +901,7 @@ mixed v$_COOKIE __attribute__ ((weak)); mixed v$_REQUEST __attribute__ ((weak)); mixed v$_ENV __attribute__ ((weak)); mixed v$_SESSION __attribute__ ((weak)); +mixed v$_KPHPSESSARR __attribute__ ((weak)); mixed v$argc __attribute__ ((weak)); mixed v$argv __attribute__ ((weak)); @@ -1472,7 +1472,8 @@ static void reset_superglobals() { hard_reset_var(v$_REQUEST, array()); hard_reset_var(v$_ENV, array()); hard_reset_var(v$_COOKIE, array()); - hard_reset_var(v$_SESSION, mixed(NULL)); // Is this correct? + hard_reset_var(v$_SESSION, array()); + hard_reset_var(v$_KPHPSESSARR, array()); dl::leave_critical_section(); } diff --git a/runtime/interface.h b/runtime/interface.h index dda1dcca3b..d5b50f3f54 100644 --- a/runtime/interface.h +++ b/runtime/interface.h @@ -152,6 +152,7 @@ extern mixed v$_COOKIE; extern mixed v$_REQUEST; extern mixed v$_ENV; extern mixed v$_SESSION; +extern mixed v$_KPHPSESSARR; const int32_t UPLOAD_ERR_OK = 0; const int32_t UPLOAD_ERR_INI_SIZE = 1; From efa217d08922ef2efa0e16cc43fe08eece46509e Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Mon, 1 Apr 2024 12:24:25 +0300 Subject: [PATCH 06/89] Change the session_start() arguments --- builtin-functions/_functions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index b4ee7d1fde..f1cc67c6cc 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -772,7 +772,7 @@ function mb_strtolower ($str ::: string, $encoding ::: string = "cp1251") ::: st function mb_strtoupper ($str ::: string, $encoding ::: string = "cp1251") ::: string; function mb_substr ($str ::: string, $start ::: int, $length ::: mixed = PHP_INT_MAX, $encoding ::: string = "cp1251") ::: string; -function session_start(?mixed $options = null) ::: bool; +function session_start($options ::: mixed = array()) ::: bool; function session_abort() ::: bool; function session_commit() ::: bool; function session_write_close() ::: bool; From c0bf68467891e87c341e6fddd706e346e8f1e310 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Mon, 1 Apr 2024 12:25:53 +0300 Subject: [PATCH 07/89] Implement basic functions of sessions: session_start, session_read --- runtime/sessions.cpp | 507 ++++++++++++++++++------------------------- runtime/sessions.h | 76 +------ 2 files changed, 209 insertions(+), 374 deletions(-) diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index 70951f06db..c45e4dabfa 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -2,30 +2,98 @@ #include #include "runtime/sessions.h" +#include "runtime/files.h" #include "runtime/serialize-functions.h" +#include "runtime/misc.h" +#include "common/wrappers/to_array.h" #include "runtime/interface.h" -#include "runtime/streams.h" #include "runtime/url.h" -#include "runtime/files.h" -#include "runtime/misc.h" -// #include "runtime/critical_section.h" namespace sessions { -SessionManager::SessionManager() { - tempdir_path = string(getenv("TMPDIR")); - sessions_path = string(tempdir_path).append("sessions/"); - if (!f$file_exists(sessions_path)) { - f$mkdir(sessions_path); +static mixed get_sparam(const char *key) noexcept; +static mixed get_sparam(const string &key) noexcept; +static void set_sparam(const char *key, const mixed &value) noexcept; +static void set_sparam(const string &key, const mixed &value) noexcept; +static void reset_sparams() noexcept; +static void initialize_sparams(const array &options) noexcept; + +static bool session_start(); +static bool session_initialize(); +static bool session_generate_id(); +static bool session_valid_id(const string &id); +static bool session_abort(); +static bool session_reset_id(); +static bool session_send_cookie(); +// static bool session_flush(); + +static bool session_open(); +static bool session_read(); +// static bool session_write(); +static void session_close(); + +static void initialize_sparams(const array &options) noexcept { + // TO-DO: reconsider it + const auto skeys = vk::to_array>({ + {"read_and_close", false}, + {"save_path", string(getenv("TMPDIR")).append("sessions/")}, + {"name", string("PHPSESSID")}, + {"gc_maxlifetime", 1440}, + {"cookie_path", string("/")}, + {"cookie_lifetime", 0}, + {"cookie_domain", string("")}, + {"cookie_secure", false}, + {"cookie_httponly", false} + }); + + reset_sparams(); + + for (const auto& it : skeys) { + if (options.isset(string(it.first))) { + set_sparam(it.first, options.get_value(string(it.first))); + continue; + } + set_sparam(it.first, mixed(it.second)); + } +} + +static void print_sparams() noexcept { + fprintf(stdout, "\nKPHPSESSARR is:\n"); + for (auto it = v$_KPHPSESSARR.as_array().begin(); it != v$_KPHPSESSARR.as_array().end(); ++it) { + fprintf(stdout, "%s : %s\n", it.get_key().to_string().c_str(), it.get_value().to_string().c_str()); + } + + fprintf(stdout, "\n%s\n", f$serialize(v$_KPHPSESSARR.as_array()).c_str()); +} + +static void reset_sparams() noexcept { + v$_KPHPSESSARR = array(); +} + +static mixed get_sparam(const char *key) noexcept { + return get_sparam(string(key)); +} + +static mixed get_sparam(const string &key) noexcept { + if (!v$_KPHPSESSARR.as_array().isset(key)) { + return false; } + return v$_KPHPSESSARR.as_array().get_value(key); } -static SessionManager &SM = vk::singleton::get(); +static void set_sparam(const char *key, const mixed &value) noexcept { + set_sparam(string(key), value); +} + +static void set_sparam(const string &key, const mixed &value) noexcept { + v$_KPHPSESSARR.as_array().emplace_value(key, value); +} static bool session_valid_id(const string &id) { if (id.empty()) { return false; } + bool result = true; for (auto i = (string("sess_").size()); i < id.size(); ++i) { if (!((id[i] >= 'a' && id[i] <= 'z') @@ -40,395 +108,234 @@ static bool session_valid_id(const string &id) { return result; } -static bool generate_session_id() { - if (!SM.get_session_status()) { - return false; - } +static bool session_generate_id() { string id = f$uniqid(string("sess_")); - if (!session_valid_id(id)) { php_warning("Failed to create new ID\n"); return false; } - SM.set_session_id(id); + set_sparam("session_id", id); return true; } -void SessionManager::session_reset_vars() { - handler = NULL; - id = false; - session_status = send_cookie = 0; - cookie_secure = cookie_httponly = 0; - cookie_lifetime = 0; - session_path = session_name = cookie_domain = false; -} - -bool SessionManager::session_write() { - if (!session_open()) { - return false; - } - f$file_put_contents(handler.to_string(), string("")); // f$ftruncate(handler, 0); - f$rewind(handler); - { // additional local scope for temprory variables tdata and tres - string tdata = f$serialize(v$_SESSION); - mixed tres = f$file_put_contents(handler.to_string(), tdata); - if (tres.is_bool() || static_cast(tres.as_int()) != tdata.size()) { - // php_warning(); - return false; - } +static bool session_abort() { + if (get_sparam("session_status").to_bool()) { + session_close(); + set_sparam("session_status", false); + return true; } - return true; + return false; } -bool SessionManager::session_read() { - if ((handler.is_null()) || (!session_open())) { - // php_warning(); - return false; +static bool session_open() { + if (get_sparam("handler").to_bool()) { + /* + if we close the file for reopening it, + the other worker may open it faster and the current process will stop + */ + return true; } - { // additional local scope for temprory variable tdata - Optional tdata; - tdata = f$file_get_contents(handler.to_string()); - if (tdata.is_false()) { - return false; - } - v$_SESSION = f$unserialize(tdata.val()); // TO-DO: check + + if (!f$file_exists(get_sparam("save_path").to_string())) { + f$mkdir(get_sparam("save_path").to_string()); + fprintf(stdout, "session_open(): successfully created dir: %s\n", get_sparam("save_path").to_string().c_str()); } - fprintf(stdout, "successfully read the session data with id %s\n", SM.get_session_id().val().c_str()); - return true; -} -bool SessionManager::session_open() { - if (id.is_false()) { + set_sparam("file_path", string(get_sparam("save_path").to_string()).append(get_sparam("session_id").to_string())); + set_sparam("handler", open_safe(get_sparam("file_path").to_string().c_str(), O_RDWR | O_CREAT, 0777)); + + if (get_sparam("handler").to_int() < 0) { + php_warning("Failed to open the file %s", get_sparam("file_path").to_string().c_str()); return false; } - if (!handler.is_null()) { - session_close(); - } + fprintf(stdout, "session_open(): successfully opened file: handler %lld, path %s\n", get_sparam("handler").to_int(), get_sparam("file_path").to_string().c_str()); - if (session_path.is_false()) { - // setting the session_path is only here - session_path = sessions_path; - session_path.val().append(id.val()); - session_path.val().append(".sess"); + if (flock(get_sparam("handler").to_int(), LOCK_EX) < 0) { + php_warning("Failed to lock the file %s", get_sparam("file_path").to_string().c_str()); + return false; } - if (!f$file_exists(session_path.val())) { - f$fclose(f$fopen(session_path.val(), string("w+"))); - } + fprintf(stdout, "session_open(): successfully locked file: handler %lld, path %s\n", get_sparam("handler").to_int(), get_sparam("file_path").to_string().c_str()); - { // additional local scope for temprory variable thandler - mixed thandler = f$fopen(session_path.val(), string("r+")); - handler = (!thandler.is_bool()) ? thandler : NULL; - } - if (handler.is_null()) { - return false; - } return true; } -static bool session_destroy(const string &filepath) { - - Optional ctime = f$filectime(filepath); - if (ctime.is_false()) { - php_warning("Failed to get metadata of file %s\n", filepath.c_str()); - } - - // Stream thandler = f$fopen(filepath, string("r")); - // if (!thandler) { - // php_warning("Failed to open file %s\n", filepath.c_str()); - // return false; - // } - - Optional sdata = f$file_get_contents(filepath); - if (sdata.is_false()) { - php_warning("Failed to read file %s\n", filepath.c_str()); - return false; - } - array tdata = f$unserialize(sdata.val()).to_array(); - int lifetime = 0; - if (tdata.has_key(string("gc_maxlifetime"))) { - lifetime = tdata.get_value(string("gc_maxlifetime")).to_int(); - } - if (ctime < lifetime) { - // f$fclose(thandler); - return false; +static void session_close() { + if (get_sparam("handler").to_bool()) { + close_safe(get_sparam("handler").to_int()); } - // f$fclose(thandler); - return f$unlink(filepath); + reset_sparams(); } -static bool session_gc() { - Optional> sfiles = f$scandir(SM.get_sessions_path()); - if (sfiles.is_false()) { - php_warning("Failed to scan sessions directory %s\n", SM.get_sessions_path().c_str()); +static bool session_read() { + session_open(); + struct stat buf; + if (fstat(get_sparam("handler").to_int(), &buf) < 0) { + php_warning("Failed to read session data on path %s", get_sparam("file_path").to_string().c_str()); return false; } - fprintf(stdout, "result of scandir is:\n%s\n", f$serialize(sfiles.val()).c_str()); - auto nsfiles = sfiles.val().count(); - int n = 0; - for (auto sfile = sfiles.val().cbegin(); sfile != sfiles.val().cend(); ++sfile) { - string val = sfile.get_value().to_string(); - if (val == string(".") or val == string("..")) { - continue; - } - if (session_destroy(val)) { - ++n; - } - } - if (n != nsfiles) { - php_warning("Failed to delete all files in sessions dir %s\n", SM.get_sessions_path().c_str()); - return false; - } - return true; -} -bool SessionManager::session_close(bool immediate) { - if (handler.is_null()) { + if (buf.st_size == 0) { + v$_SESSION = array(); return true; } - - if (!f$fclose(handler)) { - // php_warning(); + + char result[buf.st_size]; + dl::enter_critical_section(); + int n = read(get_sparam("handler").to_int(), result, buf.st_size); + if (n < (int)buf.st_size) { + if (n == -1) { + php_warning("Read failed"); + } else { + php_warning("Read returned less bytes than requested"); + } + dl::leave_critical_section(); return false; } - session_reset_vars(); + dl::leave_critical_section(); - if (immediate or rand() % 100 > 75) { - session_gc(); + v$_SESSION = f$unserialize(string(result, n)); + fprintf(stdout, "session_open(): successfully read file: handler %lld, path %s\n", get_sparam("handler").to_int(), get_sparam("file_path").to_string().c_str()); + fprintf(stdout, "_SESSION is: \n"); + for (auto it = v$_SESSION.as_array().begin(); it != v$_SESSION.as_array().end(); ++it) { + fprintf(stdout, "%s : %s\n", it.get_key().to_string().c_str(), it.get_value().to_string().c_str()); } return true; } -static bool session_abort() { - if (SM.get_session_status()) { - SM.session_close(); - SM.set_session_status(0); - return true; - } - return false; -} - -static bool session_flush(bool write = false) { - if (!SM.get_session_status()) { - return false; - } - bool result = true; - if (write && !SM.session_write()) { - php_warning("Failed to write session data %s\n", SM.get_session_path().val().c_str()); - result = false; - } - - SM.set_session_status(0); - SM.session_close(); - return result; -} +// static bool session_write() { +// session_open(); +// // pass +// return true; +// } -static bool send_cookies() { - // 1. check whether headers were sent or not - // pass: need headers_sent() - - // 2. check session_name for deprecated symbols, if it's user supplied - if (strpbrk(SM.get_session_name().val().c_str(), "=,;.[ \t\r\n\013\014") != NULL) { +static bool session_send_cookie() { + if (strpbrk(get_sparam("name").to_string().c_str(), "=,;.[ \t\r\n\013\014") != NULL) { php_warning("session.name cannot contain any of the following '=,;.[ \\t\\r\\n\\013\\014'"); return false; } - // 3. encode session_name, if it's user supplied - int expire = SM.get_session_lifetime(); + int expire = get_sparam("cookie_lifetime").to_int(); if (expire > 0) { expire += std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); } - string domain = !SM.get_cookie_domain().is_false() ? SM.get_cookie_domain().val() : string(""); - f$setcookie(SM.get_session_name().val(), f$urlencode(SM.get_session_id().val()), expire, string("/"), domain, SM.get_cookie_secure(), SM.get_cookie_httponly()); - fprintf(stdout, "successfully sent cookies for id %s\n", SM.get_session_id().val().c_str()); + string domain = get_sparam("cookie_domain").to_string(); + bool secure = get_sparam("cookie_secure").to_bool(); + bool httponly = get_sparam("cookie_httponly").to_bool(); + string sid = f$urlencode(get_sparam("session_id").to_string()); + string path = get_sparam("cookie_path").to_string(); + string name = get_sparam("name").to_string(); + + f$setcookie(name, sid, expire, path, domain, secure, httponly); return true; } static bool session_reset_id() { - if (SM.get_session_id().is_false()) { + if (!get_sparam("session_id").to_bool()) { php_warning("Cannot set session ID - session ID is not initialized"); return false; } - if (SM.get_send_cookie()) { - send_cookies(); - SM.set_send_cookie(0); + if (get_sparam("send_cookie").to_bool()) { + session_send_cookie(); + set_sparam("send_cookie", true); } - return true; } static bool session_initialize() { - SM.set_session_status(1); - - if (SM.get_session_id().is_false()) { - generate_session_id(); - if (SM.get_session_id().is_false()) { + set_sparam("session_status", true); + + if (!get_sparam("session_id").to_bool()) { + if (!session_generate_id()) { + php_warning( + "Failed to create session ID: %s (path: %s)", + get_sparam("name").to_string().c_str(), + get_sparam("save_path").to_string().c_str() + ); session_abort(); - php_warning("Failed to create session ID: %s (path: %s)", SM.get_session_name().val().c_str(), SM.get_session_path().val().c_str()); return false; } - SM.set_send_cookie(1); + set_sparam("send_cookie", true); } + fprintf(stdout, "session_initialize(): successfully generated id: %s\n", get_sparam("session_id").to_string().c_str()); - SM.session_open(); - if (SM.get_handler().is_null()) { + if (!session_open() or !session_reset_id() or !session_read()) { session_abort(); - // php_warning(); return false; } - - session_reset_id(); - if (!SM.session_read()) { - session_abort(); - php_warning("Failed to read session data: %s (path: %s)\n", SM.get_session_name().val().c_str(), SM.get_session_path().val().c_str()); - return false; - } + // session_gc() + // pass return true; } static bool session_start() { - // 1. check status of the session - if (SM.get_session_status()) { - if (SM.get_session_path().is_false()) { - php_warning("Ignoring session_start() because a session is already active (started from %s)", SM.get_session_path().val().c_str()); - } else { - php_warning("Ignoring session_start() because a session is already active"); - } + if (get_sparam("session_status").to_bool()) { + php_warning("Ignoring session_start() because a session is already active"); return false; } - SM.set_send_cookie(1); + set_sparam("send_cookie", true); - if (SM.get_session_name().is_false()) { - SM.set_session_name(string("PHPSESSID")); - } - fprintf(stdout, "session name is: %s\n", SM.get_session_name().val().c_str()); - - // 2. check id of session - if (SM.get_session_id().is_false()) { - // 4.1 try to get an id of session via it's name from _COOKIE - Optional pid{false}; - if (v$_COOKIE.as_array().has_key(SM.get_session_name().val())) { - pid = v$_COOKIE.get_value(SM.get_session_name().val()).to_string(); - fprintf(stdout, "found session id in cookies: %s\n", pid.val().c_str()); + if (!get_sparam("session_id").to_bool()) { + mixed id = false; + if (v$_COOKIE.as_array().isset(get_sparam("name").to_string())) { + id = v$_COOKIE.as_array().get_value(get_sparam("name").to_string()).to_string(); + fprintf(stdout, "Found id: %s\n", id.as_string().c_str()); } - if (!pid.is_false()) { - SM.set_send_cookie(0); - } - SM.set_session_id(pid); - } - // 5. check the id for dangerous symbols - if (!SM.get_session_id().is_false() && strpbrk(SM.get_session_id().val().c_str(), "\r\n\t <>'\"\\")) { - SM.set_session_id(false); + if (id.to_bool() && !id.to_string().empty()) { + if (!strpbrk(id.to_string().c_str(), "\r\n\t <>'\"\\")) { + fprintf(stdout, "id is valid\n"); + set_sparam("send_cookie", false); + set_sparam("session_id", id.to_string()); + } + } } - // 6. try to initialize the session if (!session_initialize()) { - SM.set_session_status(0); - SM.set_session_id(false); + set_sparam("session_status", false); + set_sparam("session_id", false); return false; } + return true; } +// static bool session_flush() { +// if (!get_sparam("session_status").to_bool()) { +// return false; +// } + +// session_write(); +// set_sparam("session_status", false); +// return true; +// } + } // namespace sessions bool f$session_start(const array &options) { - // 1. check status of session - if (sessions::SM.get_session_status()) { - if (sessions::SM.get_session_path().is_false()) { - php_warning("Ignoring session_start() because a session is already active (started from %s)", sessions::SM.get_session_path().val().c_str()); - } else { - php_warning("Ignoring session_start() because a session is already active"); - } - return true; + if (sessions::get_sparam("session_status").to_bool()) { + php_warning("Ignoring session_start() because a session is already active"); + return false; } - // 2. check whether headers were sent or not - // need headers_sent() - // pass - - bool read_and_close = false; - - // 3. parse the options arg and set it - if (!options.empty()) { - for (auto it = options.begin(); it != options.end(); ++it) { - string tkey = it.get_key().to_string(); - if (tkey == string("read_and_close")) { - read_and_close = it.get_value().to_bool(); - } else if (tkey == string("name")) { - sessions::SM.set_session_name(it.get_value().to_string()); - } else if (tkey == string("gc_maxlifetime")) { - sessions::SM.set_session_lifetime(it.get_value().to_int()); - } else if (tkey == string("cookie_path")) { - sessions::SM.set_session_path(it.get_value().to_string()); - } else if (tkey == string("cookie_lifetime")) { - sessions::SM.set_cookie_lifetime(it.get_value().to_int()); - } else if (tkey == string("cookie_domain")) { - sessions::SM.set_cookie_domain(it.get_value().to_string()); - } else if (tkey == string("cookie_secure")) { - sessions::SM.set_cookie_secure(it.get_value().to_bool()); - } else if (tkey == string("cookie_httponly")) { - sessions::SM.set_cookie_httponly(it.get_value().to_bool()); - } - } - } - fprintf(stdout, "Before session_start()\n"); + sessions::initialize_sparams(options); + sessions::print_sparams(); // to delete sessions::session_start(); - fprintf(stdout, "After session_start()\n"); - - // 4. check the status of the session - if (!sessions::SM.get_session_status()) { - // 4.1 clear cache of session variables - v$_SESSION = NULL; - // php_warning(); - return false; - } - // 5. check for read_and_close - if (read_and_close) { - sessions::session_flush(); + if (sessions::get_sparam("read_and_close").to_bool()) { + sessions::session_close(); } - return true; } -bool f$session_destroy() { - if (!sessions::SM.get_session_status()) { - php_warning("Trying to destroy uninitialized session"); - return false; - } - - if (sessions::SM.get_session_path().has_value()) { - return sessions::session_destroy(sessions::SM.get_session_path().val()); - } - sessions::SM.session_close(); - return false; -} - bool f$session_abort() { - if (!sessions::SM.get_session_status()) { - return false; - } - return sessions::session_abort(); -} - -int64_t f$session_status() { - return static_cast(sessions::SM.get_session_status()) + 1; -} - -bool f$session_write_close() { - if (!sessions::SM.get_session_status()) { + if (!sessions::get_sparam("session_status").to_bool()) { return false; } - return sessions::session_flush(1); -} - -bool $session_commit() { - return f$session_write_close(); + sessions::session_abort(); + return true; } - diff --git a/runtime/sessions.h b/runtime/sessions.h index e66a5d0c5e..61072c0992 100644 --- a/runtime/sessions.h +++ b/runtime/sessions.h @@ -1,79 +1,7 @@ #pragma once #include "runtime/kphp_core.h" -#include "common/mixin/not_copyable.h" -#include "common/smart_ptrs/singleton.h" - -namespace sessions { - -using Stream = mixed; - -class SessionManager : vk::not_copyable { -public: - bool session_open(); - bool session_close(bool immediate = false); - bool session_read(); - bool session_write(); - bool session_delete(); - - void session_reset_vars(); - - void set_session_status(const bool &status) noexcept { session_status = status; } - void set_session_path(const string &path) noexcept { session_path = path; } - void set_session_id(const Optional &sid) noexcept { id = sid; } - void set_send_cookie(const bool &val) noexcept { send_cookie = val; } - void set_session_name(const string &val) noexcept { session_name = val; } - void set_session_lifetime(const int &val) noexcept { session_lifetime = val; } - void set_handler(const Stream &stream) noexcept { handler = stream; } - - void set_cookie_domain(const string &val) noexcept { cookie_domain = val; } - void set_cookie_lifetime(const int &val) noexcept { cookie_lifetime = val; } - void set_cookie_secure(const bool &val) noexcept { cookie_secure = val; } - void set_cookie_httponly(const bool &val) noexcept { cookie_httponly = val; } - - bool get_session_status() noexcept { return session_status; } - Optional get_session_path() noexcept { return session_path; } - Optional get_session_id() noexcept { return id; } - bool get_send_cookie() noexcept { return send_cookie; } - Optional get_session_name() noexcept { return session_name; } - int get_session_lifetime() noexcept { return session_lifetime; } - Stream get_handler() noexcept {return handler; } - - string get_sessions_path() noexcept { return sessions_path; } - Optional get_cookie_domain() noexcept { return cookie_domain; } - int get_cookie_lifetime() noexcept { return cookie_lifetime; } - bool get_cookie_secure() noexcept { return cookie_secure; } - bool get_cookie_httponly() noexcept { return cookie_httponly; } - -private: - SessionManager(); - - Optional id{false}; - bool session_status{0}; - bool send_cookie{0}; - Optional session_path{false}; - Optional session_name{false}; // by default PHPSESSID (key for searching in the cookies) - int session_lifetime{1440}; - - int cookie_lifetime{0}; - Optional cookie_domain{false}; - bool cookie_secure{0}; - bool cookie_httponly{0}; - - string tempdir_path; // path to the /tmp - string sessions_path; // path to the dir with existing sessions - - Stream handler{NULL}; - - friend class vk::singleton; -}; - -} // namespace sessions +#include "runtime/interface.h" bool f$session_start(const array &options = array()); -bool f$session_abort(); -bool f$session_destroy(); -bool f$session_commit(); -bool f$session_write_close(); -int64_t f$session_status(); -Optional f$session_create_id(const string &prefix); +bool f$session_abort(); \ No newline at end of file From e6a68948750bd98edc4b5b7ab88a1e5904a4c3b2 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Tue, 2 Apr 2024 22:19:14 +0300 Subject: [PATCH 08/89] Refactor the code: add aliases for array keys. Implement other basic functions and simple logic for working with extra attributes for session files. --- builtin-functions/_functions.txt | 6 +- runtime/sessions.cpp | 390 +++++++++++++++++++++++-------- runtime/sessions.h | 13 +- 3 files changed, 305 insertions(+), 104 deletions(-) diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index f1cc67c6cc..0f83c41024 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -777,8 +777,12 @@ function session_abort() ::: bool; function session_commit() ::: bool; function session_write_close() ::: bool; function session_gc() ::: bool; -function session_status() ::: bool; +function session_status() ::: int; +function session_id($id ::: ?string = null) ::: string | false; function session_destroy() ::: bool; +function session_encode() ::: string | false; +function session_decode($data ::: string) ::: bool; +function session_get_cookie_params() ::: array; define('PHP_ROUND_HALF_UP', 123423141); define('PHP_ROUND_HALF_DOWN', 123423144); diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index c45e4dabfa..ec0b8d91fe 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "runtime/sessions.h" #include "runtime/files.h" @@ -8,6 +9,7 @@ #include "common/wrappers/to_array.h" #include "runtime/interface.h" #include "runtime/url.h" +#include "runtime/math_functions.h" namespace sessions { @@ -25,29 +27,52 @@ static bool session_valid_id(const string &id); static bool session_abort(); static bool session_reset_id(); static bool session_send_cookie(); -// static bool session_flush(); +static int session_gc(const bool &immediate); +static bool session_flush(); +static string session_encode(); +static bool session_decode(const string &data); static bool session_open(); static bool session_read(); -// static bool session_write(); +static bool session_write(); static void session_close(); -static void initialize_sparams(const array &options) noexcept { - // TO-DO: reconsider it - const auto skeys = vk::to_array>({ - {"read_and_close", false}, - {"save_path", string(getenv("TMPDIR")).append("sessions/")}, - {"name", string("PHPSESSID")}, - {"gc_maxlifetime", 1440}, - {"cookie_path", string("/")}, - {"cookie_lifetime", 0}, - {"cookie_domain", string("")}, - {"cookie_secure", false}, - {"cookie_httponly", false} - }); +constexpr static auto S_READ_CLOSE = "read_and_close"; +constexpr static auto S_ID = "session_id"; +constexpr static auto S_FD = "handler"; +constexpr static auto S_STATUS = "session_status"; +constexpr static auto S_OPENED = "is_opened"; +constexpr static auto S_DIR = "save_path"; +constexpr static auto S_PATH = "session_path"; +constexpr static auto S_NAME = "name"; +constexpr static auto S_CTIME = "session_ctime"; +constexpr static auto S_LIFETIME = "gc_maxlifetime"; +constexpr static auto S_PROBABILITY = "gc_probability"; +constexpr static auto S_DIVISOR = "gc_divisor"; +constexpr static auto S_SEND_COOKIE = "send_cookie"; +constexpr static auto C_PATH = "cookie_path"; +constexpr static auto C_LIFETIME = "cookie_lifetime"; +constexpr static auto C_DOMAIN = "cookie_domain"; +constexpr static auto C_SECURE = "cookie_secure"; +constexpr static auto C_HTTPONLY = "cookie_httponly"; + +// TO-DO: reconsider it +const auto skeys = vk::to_array>({ + {S_READ_CLOSE, false}, + {S_DIR, string(getenv("TMPDIR")).append("sessions/")}, + {S_NAME, string("PHPSESSID")}, + {S_LIFETIME, 1440}, + {S_PROBABILITY, 1}, + {S_DIVISOR, 100}, + {C_PATH, string("/")}, + {C_LIFETIME, 0}, + {C_DOMAIN, string("")}, + {C_SECURE, false}, + {C_HTTPONLY, false} +}); +static void initialize_sparams(const array &options) noexcept { reset_sparams(); - for (const auto& it : skeys) { if (options.isset(string(it.first))) { set_sparam(it.first, options.get_value(string(it.first))); @@ -57,17 +82,27 @@ static void initialize_sparams(const array &options) noexcept { } } -static void print_sparams() noexcept { - fprintf(stdout, "\nKPHPSESSARR is:\n"); - for (auto it = v$_KPHPSESSARR.as_array().begin(); it != v$_KPHPSESSARR.as_array().end(); ++it) { - fprintf(stdout, "%s : %s\n", it.get_key().to_string().c_str(), it.get_value().to_string().c_str()); +static array session_get_cookie_params() { + array result; + if (v$_KPHPSESSARR.as_array().empty()) { + php_warning("Session cookie params cannot be received when there is no active session. Returned the default params"); + result.emplace_value(string(C_PATH), skeys[6].second); + result.emplace_value(string(C_LIFETIME), skeys[7].second); + result.emplace_value(string(C_DOMAIN), skeys[8].second); + result.emplace_value(string(C_SECURE), skeys[9].second); + result.emplace_value(string(C_HTTPONLY), skeys[10].second); + } else { + result.emplace_value(string(C_PATH), get_sparam(C_PATH)); + result.emplace_value(string(C_LIFETIME), get_sparam(C_LIFETIME)); + result.emplace_value(string(C_DOMAIN), get_sparam(C_DOMAIN)); + result.emplace_value(string(C_SECURE), get_sparam(C_SECURE)); + result.emplace_value(string(C_HTTPONLY), get_sparam(C_HTTPONLY)); } - - fprintf(stdout, "\n%s\n", f$serialize(v$_KPHPSESSARR.as_array()).c_str()); + return result; } static void reset_sparams() noexcept { - v$_KPHPSESSARR = array(); + v$_KPHPSESSARR.as_array().clear(); } static mixed get_sparam(const char *key) noexcept { @@ -114,21 +149,33 @@ static bool session_generate_id() { php_warning("Failed to create new ID\n"); return false; } - set_sparam("session_id", id); + set_sparam(S_ID, id); return true; } static bool session_abort() { - if (get_sparam("session_status").to_bool()) { + if (get_sparam(S_STATUS).to_bool()) { session_close(); - set_sparam("session_status", false); return true; } return false; } +static string session_encode() { + return f$serialize(v$_SESSION.as_array()); +} + +static bool session_decode(const string &data) { + mixed buf = f$unserialize(data); + if (buf.is_bool()) { + return false; + } + v$_SESSION = buf.as_array(); + return true; +} + static bool session_open() { - if (get_sparam("handler").to_bool()) { + if (get_sparam(S_FD).to_bool()) { /* if we close the file for reopening it, the other worker may open it faster and the current process will stop @@ -136,206 +183,345 @@ static bool session_open() { return true; } - if (!f$file_exists(get_sparam("save_path").to_string())) { - f$mkdir(get_sparam("save_path").to_string()); - fprintf(stdout, "session_open(): successfully created dir: %s\n", get_sparam("save_path").to_string().c_str()); + if (!f$file_exists(get_sparam(S_DIR).to_string())) { + f$mkdir(get_sparam(S_DIR).to_string()); } - set_sparam("file_path", string(get_sparam("save_path").to_string()).append(get_sparam("session_id").to_string())); - set_sparam("handler", open_safe(get_sparam("file_path").to_string().c_str(), O_RDWR | O_CREAT, 0777)); + set_sparam(S_PATH, string(get_sparam(S_DIR).to_string()).append(get_sparam(S_ID).to_string())); + bool is_new = (!f$file_exists(get_sparam(S_PATH).to_string())) ? 1 : 0; + set_sparam(S_FD, open_safe(get_sparam(S_PATH).to_string().c_str(), O_RDWR | O_CREAT, 0777)); - if (get_sparam("handler").to_int() < 0) { - php_warning("Failed to open the file %s", get_sparam("file_path").to_string().c_str()); + if (get_sparam(S_FD).to_int() < 0) { + php_warning("Failed to open the file %s", get_sparam(S_PATH).to_string().c_str()); return false; } - fprintf(stdout, "session_open(): successfully opened file: handler %lld, path %s\n", get_sparam("handler").to_int(), get_sparam("file_path").to_string().c_str()); - if (flock(get_sparam("handler").to_int(), LOCK_EX) < 0) { - php_warning("Failed to lock the file %s", get_sparam("file_path").to_string().c_str()); + if (flock(get_sparam(S_FD).to_int(), LOCK_EX) < 0) { + php_warning("Failed to lock the file %s", get_sparam(S_PATH).to_string().c_str()); return false; } - - fprintf(stdout, "session_open(): successfully locked file: handler %lld, path %s\n", get_sparam("handler").to_int(), get_sparam("file_path").to_string().c_str()); + + // set new metadata to the file + int ret_ctime = getxattr(get_sparam(S_PATH).to_string().c_str(), S_CTIME, NULL, 0, 0, 0); + int ret_gc_lifetime = getxattr(get_sparam(S_PATH).to_string().c_str(), S_LIFETIME, NULL, 0, 0, 0); + if (is_new or ret_ctime < 0) { + // add the creation data to metadata of file + int ctime = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + setxattr(get_sparam(S_PATH).to_string().c_str(), S_CTIME, &ctime, sizeof(int), 0, 0); + } + if (ret_gc_lifetime < 0) { + int gc_maxlifetime = get_sparam(S_LIFETIME).to_int(); + setxattr(get_sparam(S_PATH).to_string().c_str(), S_LIFETIME, &gc_maxlifetime, sizeof(int), 0, 0); + } + + int is_opened = 1; + setxattr(get_sparam(S_PATH).to_string().c_str(), S_OPENED, &is_opened, sizeof(int), 0, 0); return true; } static void session_close() { - if (get_sparam("handler").to_bool()) { - close_safe(get_sparam("handler").to_int()); + if (get_sparam(S_FD).to_bool()) { + close_safe(get_sparam(S_FD).to_int()); } + int is_opened = 0; + setxattr(get_sparam(S_PATH).to_string().c_str(), S_OPENED, &is_opened, sizeof(int), 0, 0); reset_sparams(); } static bool session_read() { session_open(); struct stat buf; - if (fstat(get_sparam("handler").to_int(), &buf) < 0) { - php_warning("Failed to read session data on path %s", get_sparam("file_path").to_string().c_str()); + if (fstat(get_sparam(S_FD).to_int(), &buf) < 0) { + php_warning("Failed to read session data on path %s", get_sparam(S_PATH).to_string().c_str()); return false; } if (buf.st_size == 0) { - v$_SESSION = array(); + v$_SESSION.as_array().clear(); return true; } char result[buf.st_size]; - dl::enter_critical_section(); - int n = read(get_sparam("handler").to_int(), result, buf.st_size); - if (n < (int)buf.st_size) { + ssize_t n = read_safe(get_sparam(S_FD).to_int(), result, buf.st_size, get_sparam(S_PATH).to_string()); + if (n < buf.st_size) { if (n == -1) { php_warning("Read failed"); } else { php_warning("Read returned less bytes than requested"); } - dl::leave_critical_section(); return false; } - dl::leave_critical_section(); - - v$_SESSION = f$unserialize(string(result, n)); - fprintf(stdout, "session_open(): successfully read file: handler %lld, path %s\n", get_sparam("handler").to_int(), get_sparam("file_path").to_string().c_str()); - fprintf(stdout, "_SESSION is: \n"); - for (auto it = v$_SESSION.as_array().begin(); it != v$_SESSION.as_array().end(); ++it) { - fprintf(stdout, "%s : %s\n", it.get_key().to_string().c_str(), it.get_value().to_string().c_str()); + + if (!session_decode(string(result, n))) { + php_warning("Failed to unzerialize the data"); + return false; } return true; } -// static bool session_write() { -// session_open(); -// // pass -// return true; -// } +static bool session_write() { + session_open(); + string data = f$serialize(v$_SESSION.as_array()); + ssize_t n = write_safe(get_sparam("handler").to_int(), data.c_str(), data.size(), get_sparam("file_path").to_string()); + if (n < data.size()) { + if (n == -1) { + php_warning("Write failed"); + } else { + php_warning("Write wrote less bytes than requested"); + } + return false; + } + return true; +} static bool session_send_cookie() { - if (strpbrk(get_sparam("name").to_string().c_str(), "=,;.[ \t\r\n\013\014") != NULL) { + if (strpbrk(get_sparam(S_NAME).to_string().c_str(), "=,;.[ \t\r\n\013\014") != NULL) { php_warning("session.name cannot contain any of the following '=,;.[ \\t\\r\\n\\013\\014'"); return false; } - int expire = get_sparam("cookie_lifetime").to_int(); + int expire = get_sparam(C_LIFETIME).to_int(); if (expire > 0) { expire += std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); } - string domain = get_sparam("cookie_domain").to_string(); - bool secure = get_sparam("cookie_secure").to_bool(); - bool httponly = get_sparam("cookie_httponly").to_bool(); - string sid = f$urlencode(get_sparam("session_id").to_string()); - string path = get_sparam("cookie_path").to_string(); - string name = get_sparam("name").to_string(); + string domain = get_sparam(C_DOMAIN).to_string(); + bool secure = get_sparam(C_SECURE).to_bool(); + bool httponly = get_sparam(C_HTTPONLY).to_bool(); + string sid = f$urlencode(get_sparam(S_ID).to_string()); + string path = get_sparam(C_PATH).to_string(); + string name = get_sparam(S_NAME).to_string(); f$setcookie(name, sid, expire, path, domain, secure, httponly); return true; } static bool session_reset_id() { - if (!get_sparam("session_id").to_bool()) { + if (!get_sparam(S_ID).to_bool()) { php_warning("Cannot set session ID - session ID is not initialized"); return false; } - if (get_sparam("send_cookie").to_bool()) { + if (get_sparam(S_SEND_COOKIE).to_bool()) { session_send_cookie(); - set_sparam("send_cookie", true); + set_sparam(S_SEND_COOKIE, false); } return true; } +static bool session_expired(const string &path) { + int ctime, lifetime; + int ret_ctime = getxattr(path.c_str(), S_CTIME, &ctime, sizeof(int), 0, 0); + int ret_lifetime = getxattr(path.c_str(), S_LIFETIME, &lifetime, sizeof(int), 0, 0); + if (ret_ctime < 0 or ret_lifetime < 0) { + php_warning("Failed to get metadata of the file on path: %s", path.c_str()); + return false; + } + + int now = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + if (ctime + lifetime <= now) { + return true; + } + return false; +} + +static int session_gc(const bool &immediate = false) { + double prob = f$lcg_value() * get_sparam(S_DIVISOR).to_float(); + double s_prob = get_sparam(S_PROBABILITY).to_float(); + if ((!immediate) && ((s_prob <= 0) or (prob >= s_prob))) { + return -1; + } + + mixed s_list = f$scandir(get_sparam(S_DIR).to_string()); + if (!s_list.to_bool()) { + php_warning("Failed to scan the session directory on the save_path: %s", get_sparam(S_DIR).to_string().c_str()); + return -1; + } + + int result = 0; + for (auto s = s_list.as_array().begin(); s != s_list.as_array().end(); ++s) { + string path = s.get_value().to_string(); + if (path == string(".") or path == string("..")) { + continue; + } + path = string(get_sparam(S_DIR).to_string()).append(path); + if (path == get_sparam(S_PATH).to_string()) { + continue; + } + int is_opened; + int ret_code = getxattr(path.c_str(), S_OPENED, &is_opened, sizeof(int), 0, 0); + if (ret_code < 0) { + php_warning("Failed to get metadata of the file on path: %s", path.c_str()); + continue; + } else if (is_opened) { + // TO-DO: fix the bug with always opened tags in the session files + // continue; + } + + if (session_expired(path)) { + f$unlink(path); + ++result; + } + } + return result; +} + static bool session_initialize() { - set_sparam("session_status", true); + set_sparam(S_STATUS, true); - if (!get_sparam("session_id").to_bool()) { + if (!get_sparam(S_ID).to_bool()) { if (!session_generate_id()) { php_warning( "Failed to create session ID: %s (path: %s)", - get_sparam("name").to_string().c_str(), - get_sparam("save_path").to_string().c_str() + get_sparam(S_NAME).to_string().c_str(), + get_sparam(S_PATH).to_string().c_str() ); session_abort(); return false; } - set_sparam("send_cookie", true); + set_sparam(S_SEND_COOKIE, true); } - fprintf(stdout, "session_initialize(): successfully generated id: %s\n", get_sparam("session_id").to_string().c_str()); if (!session_open() or !session_reset_id() or !session_read()) { session_abort(); return false; } - // session_gc() - // pass + session_gc(0); return true; } static bool session_start() { - if (get_sparam("session_status").to_bool()) { + if (get_sparam(S_STATUS).to_bool()) { php_warning("Ignoring session_start() because a session is already active"); return false; } - set_sparam("send_cookie", true); + set_sparam(S_SEND_COOKIE, true); - if (!get_sparam("session_id").to_bool()) { + if (!get_sparam(S_ID).to_bool()) { mixed id = false; - if (v$_COOKIE.as_array().isset(get_sparam("name").to_string())) { - id = v$_COOKIE.as_array().get_value(get_sparam("name").to_string()).to_string(); - fprintf(stdout, "Found id: %s\n", id.as_string().c_str()); + if (v$_COOKIE.as_array().isset(get_sparam(S_NAME).to_string())) { + id = v$_COOKIE.as_array().get_value(get_sparam(S_NAME).to_string()).to_string(); } if (id.to_bool() && !id.to_string().empty()) { if (!strpbrk(id.to_string().c_str(), "\r\n\t <>'\"\\")) { - fprintf(stdout, "id is valid\n"); - set_sparam("send_cookie", false); - set_sparam("session_id", id.to_string()); + if (f$file_exists(string(get_sparam(S_DIR).to_string()).append(id.to_string()))) { + set_sparam(S_SEND_COOKIE, false); + set_sparam(S_ID, id.to_string()); + } } } } if (!session_initialize()) { - set_sparam("session_status", false); - set_sparam("session_id", false); + set_sparam(S_STATUS, false); + set_sparam(S_ID, false); return false; } return true; } -// static bool session_flush() { -// if (!get_sparam("session_status").to_bool()) { -// return false; -// } +static bool session_flush() { + if (!get_sparam(S_STATUS).to_bool()) { + return false; + } -// session_write(); -// set_sparam("session_status", false); -// return true; -// } + session_write(); + session_close(); + return true; +} } // namespace sessions bool f$session_start(const array &options) { - if (sessions::get_sparam("session_status").to_bool()) { + if (sessions::get_sparam(sessions::S_STATUS).to_bool()) { php_warning("Ignoring session_start() because a session is already active"); return false; } sessions::initialize_sparams(options); - sessions::print_sparams(); // to delete sessions::session_start(); - if (sessions::get_sparam("read_and_close").to_bool()) { + if (sessions::get_sparam(sessions::S_READ_CLOSE).to_bool()) { sessions::session_close(); } return true; } bool f$session_abort() { - if (!sessions::get_sparam("session_status").to_bool()) { + if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { return false; } sessions::session_abort(); return true; } + +Optional f$session_gc() { + if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { + php_warning("Session cannot be garbage collected when there is no active session"); + return false; + } + int result = sessions::session_gc(1); + return (result <= 0) ? Optional{false} : Optional{result}; +} + +bool f$session_write_close() { + if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { + return false; + } + sessions::session_flush(); + return true; +} + +bool f$session_commit() { + return f$session_write_close(); +} + +int64_t f$session_status() { + return sessions::get_sparam(sessions::S_STATUS).to_int() + 1; +} + +Optional f$session_encode() { + return Optional{sessions::session_encode()}; +} + +bool f$session_decode(const string &data) { + if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { + php_warning("Session data cannot be decoded when there is no active session"); + return false; + } + return sessions::session_decode(data); +} + +array f$session_get_cookie_params() { + return sessions::session_get_cookie_params(); +} + +// TO-DO: is this correct? +bool f$session_destroy() { + if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { + php_warning("Trying to destroy uninitialized session"); + return false; + } + sessions::session_close(); + return true; +} + +// TO-DO: implement function for changing id of existing session +/* +Optional f$session_id(Optional id) { + if (id.has_value() && sessions::get_sparam(sessions::S_STATUS).to_bool()) { + php_warning("Session ID cannot be changed when a session is active"); + return Optional{false}; + } + // TO-DO: check headers_sent() + if (id.has_value()) { + sessions::set_sparam(sessions::S_ID, string(id)); + } + return Optional{sessions::get_sparam(sessions::S_ID).to_string()}; +} +*/ \ No newline at end of file diff --git a/runtime/sessions.h b/runtime/sessions.h index 61072c0992..2de3ae7c14 100644 --- a/runtime/sessions.h +++ b/runtime/sessions.h @@ -4,4 +4,15 @@ #include "runtime/interface.h" bool f$session_start(const array &options = array()); -bool f$session_abort(); \ No newline at end of file +bool f$session_abort(); +bool f$session_commit(); +bool f$session_write_close(); +Optional f$session_gc(); +int64_t f$session_status(); +Optional f$session_encode(); +bool f$session_decode(const string &data); +array f$session_get_cookie_params(); + +// TO-DO: +// bool f$session_destroy(); +// Optional f$session_id(Optional id = Optional()); \ No newline at end of file From e80556ba10fab3e3140d19ca573c6776c5891a74 Mon Sep 17 00:00:00 2001 From: Alexander Polyakov Date: Wed, 10 Apr 2024 14:02:56 +0300 Subject: [PATCH 09/89] Add support of 'random_int' and 'random_bytes' builtins (#977) --- builtin-functions/_functions.txt | 2 + builtin-functions/spl.txt | 2 + runtime/exception.cpp | 1 - runtime/exception.h | 22 +++++++ runtime/math_functions.cpp | 60 ++++++++++++++++++- runtime/math_functions.h | 4 ++ tests/kphp_tester.py | 5 +- tests/phpt/dl/385_random_bytes.php | 10 ++++ tests/phpt/dl/386_random_int.php | 33 ++++++++++ .../spl/02_builtin_exceptions_1.php | 35 +++++++++++ tests/python/lib/file_utils.py | 18 ++++-- tests/python/lib/testcase.py | 4 +- 12 files changed, 185 insertions(+), 11 deletions(-) create mode 100644 tests/phpt/dl/385_random_bytes.php create mode 100644 tests/phpt/dl/386_random_int.php create mode 100644 tests/phpt/exceptions/spl/02_builtin_exceptions_1.php diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index 16077a9bc4..640ff2cf2c 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -451,6 +451,8 @@ function getrandmax() ::: int; function mt_srand ($seed ::: int = PHP_INT_MIN) ::: void; function mt_rand ($l ::: int = TODO_OVERLOAD, $r ::: int = TODO_OVERLOAD) ::: int; function mt_getrandmax() ::: int; +function random_int($l ::: int, $r ::: int) ::: int | false; +function random_bytes($length ::: int) ::: string | false; function hash_algos () ::: string[]; function hash_hmac_algos () ::: string[]; diff --git a/builtin-functions/spl.txt b/builtin-functions/spl.txt index 23634114de..f2445302c3 100644 --- a/builtin-functions/spl.txt +++ b/builtin-functions/spl.txt @@ -25,6 +25,8 @@ class RangeException extends RuntimeException {} class UnderflowException extends RuntimeException {} class UnexpectedValueException extends RuntimeException {} +class Random\RandomException extends Exception {} + // https://www.php.net/manual/en/class.arrayiterator.php // TODO: make it work with T[] arrays when generic classes are available. // For now, it only supports mixed[]. diff --git a/runtime/exception.cpp b/runtime/exception.cpp index 0915170e26..3cc90786e8 100644 --- a/runtime/exception.cpp +++ b/runtime/exception.cpp @@ -85,7 +85,6 @@ Exception new_Exception(const string &file, int64_t line, const string &message, return f$_exception_set_location(f$Exception$$__construct(Exception().alloc(), message, code), file, line); } - Exception f$err(const string &file, int64_t line, const string &code, const string &desc) { return new_Exception(file, line, (static_SB.clean() << "ERR_" << code << ": " << desc).str(), 0); } diff --git a/runtime/exception.h b/runtime/exception.h index 21b2007394..9fd7047f80 100644 --- a/runtime/exception.h +++ b/runtime/exception.h @@ -4,6 +4,8 @@ #pragma once +#include + #include "common/algorithms/hashes.h" #include "common/wrappers/string_view.h" #include "runtime/dummy-visitor-methods.h" @@ -157,6 +159,22 @@ Exception new_Exception(const string &file, int64_t line, const string &message Exception f$err(const string &file, int64_t line, const string &code, const string &desc = string()); +template +inline class_instance make_throwable(const string &file, int64_t line, int64_t code, const string &desc) noexcept { + static_assert( + std::is_base_of_v, + "Template argument must be a subtype of C$Throwable"); + + auto ci = make_instance(); + + auto *ins_ptr = ci.get(); + ins_ptr->$file = file; + ins_ptr->$line = line; + ins_ptr->$code = code; + ins_ptr->$message = desc; + + return ci; +} string f$Exception$$getMessage(const Exception &e); string f$Error$$getMessage(const Error &e); @@ -265,3 +283,7 @@ struct C$UnderflowException : public C$RuntimeException { struct C$UnexpectedValueException : public C$RuntimeException { const char *get_class() const noexcept override { return "UnexpectedValueException"; } }; + +struct C$Random$RandomException : public C$Exception { + const char *get_class() const noexcept override { return "Random\\RandomException"; } +}; diff --git a/runtime/math_functions.cpp b/runtime/math_functions.cpp index 37b3f2ffb2..bbb41c88f6 100644 --- a/runtime/math_functions.cpp +++ b/runtime/math_functions.cpp @@ -2,13 +2,21 @@ // Copyright (c) 2020 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/math_functions.h" - #include #include +#include +#include #include +#if defined(__APPLE__) +#include +#else +#include +#endif + #include "common/cycleclock.h" +#include "runtime/math_functions.h" +#include "runtime/exception.h" #include "runtime/critical_section.h" #include "runtime/string_functions.h" #include "server/php-engine-vars.h" @@ -20,6 +28,15 @@ namespace { overflow = overflow || r < x || r > static_cast(std::numeric_limits::max()); return r; } + + int64_t secure_rand_buf(char * const buf, int64_t length) noexcept { +#if defined(__APPLE__) + arc4random_buf(static_cast(buf), static_cast(length)); + return 0; +#else + return getrandom(buf, static_cast(length), 0x0); +#endif + } } // namespace int64_t f$bindec(const string &number) noexcept { @@ -209,6 +226,45 @@ int64_t f$getrandmax() noexcept { return f$mt_getrandmax(); } +Optional f$random_int(int64_t l, int64_t r) noexcept { + if (unlikely(l > r)) { + php_warning("Argument #1 ($min) must be less than or equal to argument #2 ($max)"); + return false; + } + + if (unlikely(l == r)) { + return l; + } + + try { + std::random_device rd{"/dev/urandom"}; + std::uniform_int_distribution dist{l, r}; + + return dist(rd); + } catch (const std::exception &e) { + php_warning("Source of randomness cannot be found: %s", e.what()); + return false; + } catch (...) { + php_critical_error("Unhandled exception"); + } +} + +Optional f$random_bytes(int64_t length) noexcept { + if (unlikely(length < 1)) { + php_warning("Argument #1 ($length) must be greater than 0"); + return false; + } + + string str{static_cast(length), false}; + + if (secure_rand_buf(str.buffer(), static_cast(length)) == -1) { + php_warning("Source of randomness cannot be found: %s", std::strerror(errno)); + return false; + } + + return str; +} + mixed f$abs(const mixed &v) { mixed num = v.to_numeric(); if (num.is_int()) { diff --git a/runtime/math_functions.h b/runtime/math_functions.h index 08b77e15a2..9d71585763 100644 --- a/runtime/math_functions.h +++ b/runtime/math_functions.h @@ -34,6 +34,10 @@ int64_t f$rand() noexcept; int64_t f$getrandmax() noexcept; +Optional f$random_int(int64_t l, int64_t r) noexcept; + +Optional f$random_bytes(int64_t length) noexcept; + template inline T f$min(const array &a); diff --git a/tests/kphp_tester.py b/tests/kphp_tester.py index 309913ff4e..36cd561735 100755 --- a/tests/kphp_tester.py +++ b/tests/kphp_tester.py @@ -23,6 +23,7 @@ def __init__(self, file_path, test_tmp_dir, tags, env_vars: dict, out_regexps=No self.env_vars = env_vars self.out_regexps = out_regexps self.forbidden_regexps = forbidden_regexps + self.php_version = next((tag for tag in tags if tag.startswith("php")), "php7.4") def is_ok(self): return "ok" in self.tags @@ -43,7 +44,7 @@ def is_kphp_runtime_should_not_warn(self): return "kphp_runtime_should_not_warn" in self.tags def is_php8(self): - return "php8" in self.tags + return self.php_version.startswith("php8") def make_kphp_once_runner(self, use_nocc, cxx_name): tester_dir = os.path.abspath(os.path.dirname(__file__)) @@ -51,7 +52,7 @@ def make_kphp_once_runner(self, use_nocc, cxx_name): php_script_path=self.file_path, working_dir=os.path.abspath(os.path.join(self.test_tmp_dir, "working_dir")), artifacts_dir=os.path.abspath(os.path.join(self.test_tmp_dir, "artifacts")), - php_bin=search_php_bin(php8_require=self.is_php8()), + php_bin=search_php_bin(php_version=self.php_version), extra_include_dirs=[os.path.join(tester_dir, "php_include")], vkext_dir=os.path.abspath(os.path.join(tester_dir, os.path.pardir, "objs", "vkext")), use_nocc=use_nocc, diff --git a/tests/phpt/dl/385_random_bytes.php b/tests/phpt/dl/385_random_bytes.php new file mode 100644 index 0000000000..4c5072e823 --- /dev/null +++ b/tests/phpt/dl/385_random_bytes.php @@ -0,0 +1,10 @@ +@ok php8 += 100 && $x <= 500); + + $x = random_int(-450, -200); + var_dump($x >= -450 && $x <= -200); + + $x = random_int(-124, 107); + var_dump($x >= -124 && $x <= 107); + + $x = random_int(0, PHP_INT_MAX); + var_dump($x >= 0 && $x <= PHP_INT_MAX); + + $x = random_int(-PHP_INT_MAX - 1, 0); + var_dump($x >= -PHP_INT_MAX - 1 && $x <= 0); + + $x = random_int(-PHP_INT_MAX - 1, PHP_INT_MAX); + var_dump($x >= (-PHP_INT_MAX - 1) && $x <= PHP_INT_MAX); + + var_dump(random_int(0, 0)); + var_dump(random_int(1, 1)); + var_dump(random_int(-4, -4)); + var_dump(random_int(9287167122323, 9287167122323)); + var_dump(random_int(-8548276162, -8548276162)); + + var_dump(random_int(PHP_INT_MAX, PHP_INT_MAX)); + var_dump(random_int(-PHP_INT_MAX - 1, -PHP_INT_MAX - 1)); +} + +test_random_int(); diff --git a/tests/phpt/exceptions/spl/02_builtin_exceptions_1.php b/tests/phpt/exceptions/spl/02_builtin_exceptions_1.php new file mode 100644 index 0000000000..894a6078ab --- /dev/null +++ b/tests/phpt/exceptions/spl/02_builtin_exceptions_1.php @@ -0,0 +1,35 @@ +@ok php8.2 +KPHP_REQUIRE_FUNCTIONS_TYPING=1 +getLine()}:{$e->getMessage()}:{$e->getCode()}"; +} + +function test_random_exception() { + $to_throw = new Random\RandomException(__FUNCTION__, __LINE__ + 1); + + try { + throw $to_throw; + } catch (LogicException $e) { + var_dump([__LINE__ => 'unreachable']); + } catch (RuntimeException $e) { + var_dump([__LINE__ => 'unreachable']); + } catch (Exception $e) { + var_dump([__LINE__ => fmt_throwable($e)]); + var_dump([__LINE__ => $e->getMessage()]); + } + + try { + throw $to_throw; + } catch (Random\RandomException $e) { + var_dump([__LINE__ => fmt_throwable($e)]); + var_dump([__LINE__ => $e->getMessage()]); + } +} + +test_random_exception(); diff --git a/tests/python/lib/file_utils.py b/tests/python/lib/file_utils.py index b8ef2c7d1c..6cc975a36e 100644 --- a/tests/python/lib/file_utils.py +++ b/tests/python/lib/file_utils.py @@ -4,6 +4,8 @@ import sys import shutil +_SUPPORTED_PHP_VERSIONS = ["php7.4", "php8", "php8.1", "php8.2", "php8.3"] + def _check_file(file_name, file_dir, file_checker): file_path = os.path.join(file_dir, file_name) @@ -103,9 +105,17 @@ def can_ignore_sanitizer_log(sanitizer_log_file): return ignore_sanitizer -def search_php_bin(php8_require=False): +def search_php_bin(php_version: str): if sys.platform == "darwin": return shutil.which("php") - if php8_require: - return shutil.which("php8.1") or shutil.which("php8") - return shutil.which("php7.4") + + # checking from oldest to newest versions + for spv in sorted(_SUPPORTED_PHP_VERSIONS): + if spv < php_version: + continue + + exe_path = shutil.which(spv) + if exe_path is not None: + return exe_path + + return None diff --git a/tests/python/lib/testcase.py b/tests/python/lib/testcase.py index 1490ec2434..7fe93a3268 100644 --- a/tests/python/lib/testcase.py +++ b/tests/python/lib/testcase.py @@ -280,7 +280,7 @@ class KphpCompilerAutoTestCase(BaseTestCase): def __init__(self, method_name): super().__init__(method_name) - self.require_php8 = False + self.php_version = "php7.4" @classmethod def extra_class_setup(cls): @@ -318,7 +318,7 @@ def make_kphp_once_runner(self, php_script_path): php_script_path=os.path.join(self.test_dir, php_script_path), artifacts_dir=self.kphp_server_working_dir, working_dir=self.kphp_build_working_dir, - php_bin=search_php_bin(php8_require=self.require_php8), + php_bin=search_php_bin(php_version=self.php_version), use_nocc=self.should_use_nocc(), ) self.once_runner_trash_bin.append(once_runner) From 6f2f88d9f7e6c493fee919de993eed3f6b25deb6 Mon Sep 17 00:00:00 2001 From: Alexander Polyakov Date: Fri, 12 Apr 2024 12:32:59 +0300 Subject: [PATCH 10/89] [runtime] add array concat optimization (#979) The optimization is applied in case at least one of operands is empty. In that case it eliminates copy creation and returns a shared array. --- runtime/array.inl | 10 ++++++ tests/phpt/array/032_array_concat.php | 51 +++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tests/phpt/array/032_array_concat.php diff --git a/runtime/array.inl b/runtime/array.inl index 5eb7c895c5..367e79cad4 100644 --- a/runtime/array.inl +++ b/runtime/array.inl @@ -1565,6 +1565,16 @@ void array::merge_with_recursive(const mixed &other) noexcept { template const array array::operator+(const array &other) const { + bool this_empty{this->empty()}; + bool other_empty{other.empty()}; + + // short path in case at least one array is empty + if (this_empty || other_empty) { + if (this_empty && other_empty) { return {}; } + else if (other_empty) { return *this; } + else { return other; } + } + array result(size() + other.size()); if (is_vector()) { diff --git a/tests/phpt/array/032_array_concat.php b/tests/phpt/array/032_array_concat.php new file mode 100644 index 0000000000..927e816409 --- /dev/null +++ b/tests/phpt/array/032_array_concat.php @@ -0,0 +1,51 @@ +@ok + 1, "two" => 2, "three" => 3]; + + /** @var int[] */ + $empty3 = []; + + $tmp4 = $map + $empty3; + $tmp5 = $empty3 + $map; + + $map["one"] = -1; + $tmp4["one"] = -2; + $tmp5["one"] = -3; + + var_dump($map); + var_dump($tmp4); + var_dump($tmp5); +} + +test_array_concat_empty(); From e2f04f6cf834b38f0c45de10faa88db419e11c9e Mon Sep 17 00:00:00 2001 From: Mikhail Kornaukhov <66915223+mkornaukhov03@users.noreply.github.com> Date: Fri, 19 Apr 2024 17:33:13 +0300 Subject: [PATCH 11/89] Change deb.debian.org on archive.debian.org (#984) Change deb.debian.org on archive.debian.org in github pipelines and docs --- .github/workflows/Dockerfile.buster | 2 +- .../compiling-kphp-from-sources.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Dockerfile.buster b/.github/workflows/Dockerfile.buster index 09f667fd8d..908dbaed10 100644 --- a/.github/workflows/Dockerfile.buster +++ b/.github/workflows/Dockerfile.buster @@ -5,7 +5,7 @@ COPY tests/python/requirements.txt /tmp/ RUN apt-get update && \ apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget lsb-release && \ - echo "deb https://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ + echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \ wget -qO - https://packages.sury.org/php/apt.gpg | apt-key add - && \ diff --git a/docs/kphp-internals/developing-and-extending-kphp/compiling-kphp-from-sources.md b/docs/kphp-internals/developing-and-extending-kphp/compiling-kphp-from-sources.md index 7f460881dd..19ba70f813 100644 --- a/docs/kphp-internals/developing-and-extending-kphp/compiling-kphp-from-sources.md +++ b/docs/kphp-internals/developing-and-extending-kphp/compiling-kphp-from-sources.md @@ -33,7 +33,7 @@ apt-get update # utils for adding repositories apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget lsb-release # for newest cmake package -echo "deb https://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list +echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list # for curl-kphp-vk, libuber-h3-dev packages and kphp-timelib wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list From 30dc9f9855209416887966d2819e3f9d0293f3e2 Mon Sep 17 00:00:00 2001 From: Mikhail Kornaukhov <66915223+mkornaukhov03@users.noreply.github.com> Date: Mon, 22 Apr 2024 12:47:38 +0300 Subject: [PATCH 12/89] Fix KPHP_GLOBAL_SPLIT_CNT option (#980) * Increase possible parts_cnt & fix assert * Replace assert with kphp_assert and kphp_error --- compiler/code-gen/files/vars-cpp.cpp | 5 ++++- compiler/kphp2cpp.cpp | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/code-gen/files/vars-cpp.cpp b/compiler/code-gen/files/vars-cpp.cpp index 30814647ce..e206c3b9ff 100644 --- a/compiler/code-gen/files/vars-cpp.cpp +++ b/compiler/code-gen/files/vars-cpp.cpp @@ -165,7 +165,10 @@ static void compile_vars_part(CodeGenerator &W, const std::vector &vars, VarsCpp::VarsCpp(std::vector &&max_dep_levels, size_t parts_cnt) : max_dep_levels_(std::move(max_dep_levels)) , parts_cnt_(parts_cnt) { - kphp_assert (1 <= parts_cnt_ && parts_cnt_ <= 1024); + kphp_assert(1 <= parts_cnt_); + kphp_error(parts_cnt_ <= G->settings().globals_split_count.get(), + fmt_format("Too many globals initialization .cpp files({}) for the specified globals_split_count({})", parts_cnt_, + G->settings().globals_split_count.get())); } void VarsCpp::compile(CodeGenerator &W) const { diff --git a/compiler/kphp2cpp.cpp b/compiler/kphp2cpp.cpp index 9d685d4d88..48718d9fd3 100644 --- a/compiler/kphp2cpp.cpp +++ b/compiler/kphp2cpp.cpp @@ -236,7 +236,7 @@ int main(int argc, char *argv[]) { parser.add("Threads number for the transpilation", settings->threads_count, 't', "threads-count", "KPHP_THREADS_COUNT", std::to_string(get_default_threads_count())); parser.add("Count of global variables per dedicated .cpp file. Lowering it could decrease compilation time", settings->globals_split_count, - "globals-split-count", "KPHP_GLOBALS_SPLIT_COUNT", "1024"); + "globals-split-count", "KPHP_GLOBALS_SPLIT_COUNT", "1536"); parser.add("Builtin tl schema. Incompatible with lib mode", settings->tl_schema_file, 'T', "tl-schema", "KPHP_TL_SCHEMA"); parser.add("Generate storers and fetchers for internal tl functions", settings->gen_tl_internals, From e56cfd3c1f40c551015db602b55ef9bb147a7ca7 Mon Sep 17 00:00:00 2001 From: Alexander Polyakov Date: Fri, 26 Apr 2024 20:42:48 +0300 Subject: [PATCH 13/89] Optionally collect metrics from rpc requests and responses (#982) This commit adds two flags to both typed and untyped RPC TL query builtins. 'requests_extra_info' parameter collects extra info about sent RPC requests: 'request_size'. 'need_responses_extra_info' tells the runtime to collect extra info about RPC response associated with the RPC request. This info may later be extracted with new 'extract_kphp_rpc_response_extra_info' builtin. --- builtin-functions/_functions.txt | 25 +++- runtime/rpc.cpp | 79 +++++++++-- runtime/rpc.h | 11 +- runtime/rpc_extra_info.cpp | 24 ++++ runtime/rpc_extra_info.h | 41 ++++++ runtime/runtime.cmake | 1 + runtime/typed_rpc.cpp | 14 +- runtime/typed_rpc.h | 65 +++++---- tests/python/tests/rpc/__init__.py | 0 .../tests/rpc/php/VK/TL/RpcFunction.php | 33 +++++ .../tests/rpc/php/VK/TL/RpcResponse.php | 49 +++++++ .../VK/TL/_common/Types/rpcReqResultExtra.php | 125 ++++++++++++++++++ .../VK/TL/_common/Types/rpcResponseError.php | 61 +++++++++ .../VK/TL/_common/Types/rpcResponseHeader.php | 66 +++++++++ .../php/VK/TL/_common/Types/rpcResponseOk.php | 56 ++++++++ .../php/VK/TL/engine/Functions/engine_pid.php | 68 ++++++++++ .../VK/TL/engine/Functions/engine_sleep.php | 72 ++++++++++ .../VK/TL/engine/Functions/engine_stat.php | 68 ++++++++++ .../VK/TL/engine/Functions/engine_version.php | 68 ++++++++++ .../tests/rpc/php/VK/TL/net/Types/net_pid.php | 36 +++++ tests/python/tests/rpc/php/index.php | 88 ++++++++++++ tests/python/tests/rpc/test_rpc_extra_info.py | 57 ++++++++ vkext/vkext-rpc-tl-serialization.cpp | 4 +- vkext/vkext.stub.php | 8 +- vkext/vkext_arginfo.h | 4 +- vkext/vkext_legacy_arginfo.h | 4 +- 26 files changed, 1072 insertions(+), 55 deletions(-) create mode 100644 runtime/rpc_extra_info.cpp create mode 100644 runtime/rpc_extra_info.h create mode 100644 tests/python/tests/rpc/__init__.py create mode 100644 tests/python/tests/rpc/php/VK/TL/RpcFunction.php create mode 100644 tests/python/tests/rpc/php/VK/TL/RpcResponse.php create mode 100644 tests/python/tests/rpc/php/VK/TL/_common/Types/rpcReqResultExtra.php create mode 100644 tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseError.php create mode 100644 tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseHeader.php create mode 100644 tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseOk.php create mode 100644 tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_pid.php create mode 100644 tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_sleep.php create mode 100644 tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_stat.php create mode 100644 tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_version.php create mode 100644 tests/python/tests/rpc/php/VK/TL/net/Types/net_pid.php create mode 100644 tests/python/tests/rpc/php/index.php create mode 100644 tests/python/tests/rpc/test_rpc_extra_info.py diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index 640ff2cf2c..a191a3120b 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -916,6 +916,27 @@ final class RpcConnection { private function __construct(); } +/** + * 'KphpRpcRequestsExtraInfo' is a builtin KPHP class. It may accumulate extra information + * about RPC requests sent in both typed and untyped versions of rpc_tl_query builtins. + */ +final class KphpRpcRequestsExtraInfo { + /** + * 'get' returns an array of extra information (request size) about sent RPC requests. + * + * @return tuple(int)[] + */ + public function get (); +} + +/** + * 'extract_kphp_rpc_response_extra_info' function takes request ID and returns: + * 1. 'null' in case there is no extra information about the request; + * 2. 'tuple(response size, response time)' in case we got a response associated with + * the request ID. + */ +function extract_kphp_rpc_response_extra_info ($resumable_id ::: int) ::: ?tuple(int, float); + /** rpc store **/ function new_rpc_connection ($str ::: string, $port ::: int, $actor_id ::: mixed = 0, $timeout ::: float = 0.3, $connect_timeout ::: float = 0.3, $reconnect_timeout ::: float = 17.0) ::: \RpcConnection; // TODO: make actor_id int function store_gzip_pack_threshold ($pack_threshold_bytes ::: int) ::: void; @@ -956,7 +977,7 @@ function rpc_wait_concurrently ($request_id ::: int) ::: bool; function rpc_mc_parse_raw_wildcard_with_flags_to_array ($raw_result ::: string, &$result ::: array) ::: bool; function rpc_tl_query_one (\RpcConnection $rpc_conn, $arr ::: mixed, $timeout ::: float = -1.0) ::: int; -function rpc_tl_query (\RpcConnection $rpc_conn, $arr ::: array, $timeout ::: float = -1.0, $ignore_answer ::: bool = false) ::: int[]; +function rpc_tl_query (\RpcConnection $rpc_conn, $arr ::: array, $timeout ::: float = -1.0, $ignore_answer ::: bool = false, \KphpRpcRequestsExtraInfo $requests_extra_info = null, $need_responses_extra_info ::: bool = false) ::: int[]; /** @kphp-extern-func-info resumable */ function rpc_tl_query_result_one ($query_id ::: int) ::: mixed[]; /** @kphp-extern-func-info resumable */ @@ -984,7 +1005,7 @@ interface RpcResponse { /** @kphp-extern-func-info tl_common_h_dep */ function typed_rpc_tl_query_one (\RpcConnection $connection, @tl\RpcFunction $query_function, $timeout ::: float = -1.0) ::: int; /** @kphp-extern-func-info tl_common_h_dep */ -function typed_rpc_tl_query (\RpcConnection $connection, @tl\RpcFunction[] $query_functions, $timeout ::: float = -1.0, $ignore_answer ::: bool = false) ::: int[]; +function typed_rpc_tl_query (\RpcConnection $connection, @tl\RpcFunction[] $query_functions, $timeout ::: float = -1.0, $ignore_answer ::: bool = false, \KphpRpcRequestsExtraInfo $requests_extra_info = null, $need_responses_extra_info ::: bool = false) ::: int[]; /** @kphp-extern-func-info tl_common_h_dep resumable */ function typed_rpc_tl_query_result_one (int $query_id) ::: @tl\RpcResponse; /** @kphp-extern-func-info tl_common_h_dep resumable */ diff --git a/runtime/rpc.cpp b/runtime/rpc.cpp index b653190262..27e7188865 100644 --- a/runtime/rpc.cpp +++ b/runtime/rpc.cpp @@ -665,7 +665,8 @@ static void process_rpc_timeout(kphp_event_timer *timer) { return process_rpc_timeout(timer->wakeup_extra, false); } -int64_t rpc_send(const class_instance &conn, double timeout, bool ignore_answer) { +int64_t rpc_send_impl(const class_instance &conn, double timeout, rpc_request_extra_info_t &req_extra_info, bool collect_resp_extra_info, + bool ignore_answer) { if (unlikely (conn.is_null() || conn.get()->host_num < 0)) { php_warning("Wrong RpcConnection specified"); return -1; @@ -694,6 +695,10 @@ int64_t rpc_send(const class_instance &conn, double timeout, bo memcpy(p + sizeof(RpcHeaders) + extra_headers_size, rpc_payload_start, rpc_payload_size); slot_id_t q_id = rpc_send_query(conn.get()->host_num, p, static_cast(request_size), timeout_convert_to_ms(timeout)); + + // request's statistics + req_extra_info = rpc_request_extra_info_t{request_size}; + if (q_id <= 0) { return -1; } @@ -722,7 +727,10 @@ int64_t rpc_send(const class_instance &conn, double timeout, bo sizeof(rpc_request) * (rpc_requests_size - (rpc_first_unfinished_request_id - rpc_first_array_request_id))); rpc_first_array_request_id = rpc_first_unfinished_request_id; } else { - rpc_requests = static_cast (dl::reallocate(rpc_requests, sizeof(rpc_request) * 2 * rpc_requests_size, sizeof(rpc_request) * rpc_requests_size)); + rpc_requests = static_cast(dl::reallocate( + rpc_requests, + sizeof(rpc_request) * 2 * rpc_requests_size, + sizeof(rpc_request) * rpc_requests_size)); rpc_requests_size *= 2; } } @@ -739,6 +747,11 @@ int64_t rpc_send(const class_instance &conn, double timeout, bo kphp_tracing::on_rpc_query_send(q_id, cur->actor_or_port, cur->function_magic, static_cast(request_size), send_timestamp, ignore_answer); } + // response's metrics + if (collect_resp_extra_info) { + rpc_responses_extra_info_map.emplace_value(cur->resumable_id, rpc_response_extra_info_status_t::NOT_READY, rpc_response_extra_info_t{0, send_timestamp}); + } + if (ignore_answer) { int64_t resumable_id = cur->resumable_id; process_rpc_timeout(q_id, true); @@ -766,7 +779,8 @@ void f$rpc_flush() { } int64_t f$rpc_send(const class_instance &conn, double timeout) { - int64_t request_id = rpc_send(conn, timeout); + rpc_request_extra_info_t _{}; + int64_t request_id = rpc_send_impl(conn, timeout, _, false); if (request_id <= 0) { return 0; } @@ -776,7 +790,8 @@ int64_t f$rpc_send(const class_instance &conn, double timeout) } int64_t f$rpc_send_noflush(const class_instance &conn, double timeout) { - int64_t request_id = rpc_send(conn, timeout); + rpc_request_extra_info_t _{}; + int64_t request_id = rpc_send_impl(conn, timeout, _, false); if (request_id <= 0) { return 0; } @@ -784,6 +799,8 @@ int64_t f$rpc_send_noflush(const class_instance &conn, double t return request_id; } + + void process_rpc_answer(int32_t request_id, char *result, int32_t result_len) { rpc_request *request = get_rpc_request(request_id); @@ -801,6 +818,15 @@ void process_rpc_answer(int32_t request_id, char *result, int32_t result_len) { int64_t resumable_id = request->resumable_id; request->resumable_id = -1; + { // response's metrics + const auto resp_timestamp = std::chrono::duration{std::chrono::system_clock::now().time_since_epoch()}.count(); + if (rpc_responses_extra_info_map.isset(resumable_id)) { + auto &resp_extra_info = rpc_responses_extra_info_map[resumable_id]; + resp_extra_info.second = {result_len, resp_timestamp - std::get<1>(resp_extra_info.second)}; + resp_extra_info.first = rpc_response_extra_info_status_t::READY; + } + } + if (request->timer) { remove_event_timer(request->timer); } @@ -1123,7 +1149,8 @@ array fetch_function(const class_instance &rpc_query) { return new_tl_object; } -int64_t rpc_tl_query_impl(const class_instance &c, const mixed &tl_object, double timeout, bool ignore_answer, bool bytes_estimating, size_t &bytes_sent, bool flush) { +int64_t rpc_tl_query_impl(const class_instance &c, const mixed &tl_object, double timeout, rpc_request_extra_info_t &req_extra_info, + bool collect_resp_extra_info, bool ignore_answer, bool bytes_estimating, size_t &bytes_sent, bool flush) { f$rpc_clean(); class_instance rpc_query = store_function(tl_object); @@ -1131,6 +1158,7 @@ int64_t rpc_tl_query_impl(const class_instance &c, const mixed rpc_query.destroy(); CurException = Optional{}; } + if (rpc_query.is_null()) { return 0; } @@ -1138,31 +1166,38 @@ int64_t rpc_tl_query_impl(const class_instance &c, const mixed if (bytes_estimating) { estimate_and_flush_overflow(bytes_sent); } - int64_t query_id = rpc_send(c, timeout, ignore_answer); + + int64_t query_id = rpc_send_impl(c, timeout, req_extra_info, collect_resp_extra_info, ignore_answer); if (query_id <= 0) { return 0; } + if (unlikely(kphp_tracing::cur_trace_level >= 2)) { kphp_tracing::on_rpc_query_provide_details_after_send({}, tl_object); } + if (flush) { f$rpc_flush(); } + if (ignore_answer) { return -1; } + if (dl::query_num != rpc_tl_results_last_query_num) { rpc_tl_results_last_query_num = dl::query_num; } + rpc_query.get()->query_id = query_id; RpcPendingQueries::get().save(rpc_query); - + return query_id; } int64_t f$rpc_tl_query_one(const class_instance &c, const mixed &tl_object, double timeout) { size_t bytes_sent = 0; - return rpc_tl_query_impl(c, tl_object, timeout, false, false, bytes_sent, true); + rpc_request_extra_info_t _{}; + return rpc_tl_query_impl(c, tl_object, timeout, _, false, false, false, bytes_sent, true); } int64_t f$rpc_tl_pending_queries_count() { @@ -1208,18 +1243,35 @@ bool f$rpc_mc_parse_raw_wildcard_with_flags_to_array(const string &raw_result, a return true; } -array f$rpc_tl_query(const class_instance &c, const array &tl_objects, double timeout, bool ignore_answer) { - array result(tl_objects.size()); +array f$rpc_tl_query(const class_instance &c, const array &tl_objects, double timeout, bool ignore_answer, + class_instance requests_extra_info, bool need_responses_extra_info) { + if (ignore_answer && need_responses_extra_info) { + php_warning("Both $ignore_answer and $need_responses_extra_info are 'true'. Can't collect metrics for ignored answers"); + } + size_t bytes_sent = 0; + bool collect_resp_extra_info = !ignore_answer && need_responses_extra_info; + array queries{tl_objects.size()}; + array req_extra_info_arr{tl_objects.size()}; + for (auto it = tl_objects.begin(); it != tl_objects.end(); ++it) { - int64_t query_id = rpc_tl_query_impl(c, it.get_value(), timeout, ignore_answer, true, bytes_sent, false); - result.set_value(it.get_key(), query_id); + rpc_request_extra_info_t req_ei{}; + + int64_t query_id = rpc_tl_query_impl(c, it.get_value(), timeout, req_ei, collect_resp_extra_info, ignore_answer, true, bytes_sent, false); + + queries.set_value(it.get_key(), query_id); + req_extra_info_arr.set_value(it.get_key(), std::move(req_ei)); } + if (bytes_sent > 0) { f$rpc_flush(); } - return result; + if (!requests_extra_info.is_null()) { + requests_extra_info->extra_info_arr_ = std::move(req_extra_info_arr); + } + + return queries; } @@ -1402,6 +1454,7 @@ static void reset_rpc_global_vars() { hard_reset_var(rpc_data_copy_backup); hard_reset_var(rpc_request_need_timer); fail_rpc_on_int32_overflow = false; + hard_reset_var(rpc_responses_extra_info_map); } void init_rpc_lib() { diff --git a/runtime/rpc.h b/runtime/rpc.h index 51cfc4b6b8..7cf9844758 100644 --- a/runtime/rpc.h +++ b/runtime/rpc.h @@ -13,6 +13,7 @@ #include "runtime/net_events.h" #include "runtime/resumable.h" #include "runtime/to-array-processor.h" +#include "runtime/rpc_extra_info.h" DECLARE_VERBOSITY(rpc); @@ -81,6 +82,7 @@ int32_t rpc_fetch_int(); int64_t f$fetch_int(); int64_t f$fetch_lookup_int(); + string f$fetch_lookup_data(int64_t x4_bytes_length); int64_t f$fetch_long(); @@ -197,7 +199,9 @@ bool f$rpc_clean(bool is_error = false); bool rpc_store(bool is_error = false); int64_t f$rpc_send(const class_instance &conn, double timeout = -1.0); -int64_t rpc_send(const class_instance &conn, double timeout, bool ignore_answer = false); + +int64_t rpc_send_impl(const class_instance &conn, double timeout, rpc_request_extra_info_t &req_extra_info, bool collect_resp_extra_info, + bool ignore_answer = false); int64_t f$rpc_send_noflush(const class_instance &conn, double timeout = -1.0); @@ -208,8 +212,8 @@ Optional f$rpc_get(int64_t request_id, double timeout = -1.0); Optional f$rpc_get_synchronously(int64_t request_id); bool rpc_get_and_parse(int64_t request_id, double timeout); -bool f$rpc_get_and_parse(int64_t request_id, double timeout = -1.0); +bool f$rpc_get_and_parse(int64_t request_id, double timeout = -1.0); int64_t f$rpc_queue_create(); @@ -248,7 +252,8 @@ int64_t f$rpc_tl_query_one(const class_instance &c, const mixed int64_t f$rpc_tl_pending_queries_count(); bool f$rpc_mc_parse_raw_wildcard_with_flags_to_array(const string &raw_result, array &result); -array f$rpc_tl_query(const class_instance &c, const array &tl_objects, double timeout = -1.0, bool ignore_answer = false); +array f$rpc_tl_query(const class_instance &c, const array &tl_objects, double timeout = -1.0, bool ignore_answer = false, + class_instance requests_extra_info = {}, bool need_responses_extra_info = false); array f$rpc_tl_query_result_one(int64_t query_id); diff --git a/runtime/rpc_extra_info.cpp b/runtime/rpc_extra_info.cpp new file mode 100644 index 0000000000..794e7b7c56 --- /dev/null +++ b/runtime/rpc_extra_info.cpp @@ -0,0 +1,24 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime/rpc_extra_info.h" + + +array> rpc_responses_extra_info_map; + +array f$KphpRpcRequestsExtraInfo$$get(class_instance v$this) { + return v$this->extra_info_arr_; +} + +Optional f$extract_kphp_rpc_response_extra_info(std::int64_t resumable_id) { + const auto *resp_extra_info_ptr = rpc_responses_extra_info_map.find_value(resumable_id); + + if (resp_extra_info_ptr == nullptr || resp_extra_info_ptr->first == rpc_response_extra_info_status_t::NOT_READY) { + return {}; + } + + const auto res = resp_extra_info_ptr->second; + rpc_responses_extra_info_map.unset(resumable_id); + return res; +} diff --git a/runtime/rpc_extra_info.h b/runtime/rpc_extra_info.h new file mode 100644 index 0000000000..db55ca3bad --- /dev/null +++ b/runtime/rpc_extra_info.h @@ -0,0 +1,41 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "runtime/kphp_core.h" +#include "runtime/dummy-visitor-methods.h" +#include "runtime/refcountable_php_classes.h" +#include "common/algorithms/hashes.h" +#include "common/wrappers/string_view.h" + + +using rpc_request_extra_info_t = std::tuple; // tuple(request_size) +using rpc_response_extra_info_t = std::tuple; // tuple(response_size, response_time) +enum class rpc_response_extra_info_status_t : std::uint8_t { NOT_READY, READY }; + +extern array> rpc_responses_extra_info_map; + +struct C$KphpRpcRequestsExtraInfo final : public refcountable_php_classes, private DummyVisitorMethods { + using DummyVisitorMethods::accept; + + array extra_info_arr_; + + C$KphpRpcRequestsExtraInfo() = default; + + const char *get_class() const noexcept { + return R"(KphpRpcRequestsExtraInfo)"; + } + + int get_hash() const noexcept { + return static_cast(vk::std_hash(vk::string_view(C$KphpRpcRequestsExtraInfo::get_class()))); + } +}; + +array f$KphpRpcRequestsExtraInfo$$get(class_instance v$this); + +Optional f$extract_kphp_rpc_response_extra_info(std::int64_t resumable_id); diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index f7e0ab8222..fdcc227b92 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -100,6 +100,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ regexp.cpp resumable.cpp rpc.cpp + rpc_extra_info.cpp serialize-functions.cpp storage.cpp streams.cpp diff --git a/runtime/typed_rpc.cpp b/runtime/typed_rpc.cpp index 2133dd2075..867ade841c 100644 --- a/runtime/typed_rpc.cpp +++ b/runtime/typed_rpc.cpp @@ -137,13 +137,9 @@ class typed_rpc_tl_query_result_resumable : public Resumable { }; } // namespace -int64_t typed_rpc_tl_query_impl(const class_instance &connection, - const RpcRequest &req, - double timeout, - bool ignore_answer, - bool bytes_estimating, - size_t &bytes_sent, - bool flush) { +int64_t +typed_rpc_tl_query_impl(const class_instance &connection, const RpcRequest &req, double timeout, rpc_request_extra_info_t &req_extra_info, + bool collect_resp_extra_info, bool ignore_answer, bool bytes_estimating, size_t &bytes_sent, bool flush) { f$rpc_clean(); if (req.empty()) { php_warning("Writing rpc query error: query function is null"); @@ -158,7 +154,9 @@ int64_t typed_rpc_tl_query_impl(const class_instance &connectio if (bytes_estimating) { estimate_and_flush_overflow(bytes_sent); } - const int64_t query_id = rpc_send(connection, timeout, ignore_answer); + + const int64_t query_id = rpc_send_impl(connection, timeout, req_extra_info, collect_resp_extra_info, ignore_answer); + if (query_id <= 0) { return 0; } diff --git a/runtime/typed_rpc.h b/runtime/typed_rpc.h index 27894783fb..f756084a51 100644 --- a/runtime/typed_rpc.h +++ b/runtime/typed_rpc.h @@ -4,6 +4,7 @@ #pragma once #include "runtime/kphp_core.h" +#include "runtime/rpc_extra_info.h" #include "runtime/tl/rpc_function.h" #include "runtime/tl/rpc_tl_query.h" #include "runtime/tl/rpc_request.h" @@ -14,6 +15,8 @@ struct C$RpcConnection; int64_t typed_rpc_tl_query_impl(const class_instance &connection, const RpcRequest &req, double timeout, + rpc_request_extra_info_t &req_extra_info, + bool collect_resp_extra_info, bool ignore_answer, bool bytes_estimating, size_t &bytes_sent, @@ -23,61 +26,77 @@ class_instance typed_rpc_tl_query_result_one_impl(int64_t q const RpcErrorFactory &error_factory); array> typed_rpc_tl_query_result_impl(const array &query_ids, - const RpcErrorFactory &error_factory); + const RpcErrorFactory &error_factory); array> typed_rpc_tl_query_result_synchronously_impl(const array &query_ids, - const RpcErrorFactory &error_factory); + const RpcErrorFactory &error_factory); template -int64_t f$typed_rpc_tl_query_one(const class_instance &connection, - const class_instance &query_function, - double timeout = -1.0) { - static_assert(std::is_base_of::value, "Unexpected type"); - static_assert(std::is_same::value, "Unexpected type"); +int64_t f$typed_rpc_tl_query_one(const class_instance &connection, const class_instance &query_function, double timeout = -1.0) { + static_assert(std::is_base_of_v, "Unexpected type"); + static_assert(std::is_same_v, "Unexpected type"); size_t bytes_sent = 0; - return typed_rpc_tl_query_impl(connection, R{query_function}, timeout, false, false, bytes_sent, true); + rpc_request_extra_info_t _{}; + return typed_rpc_tl_query_impl(connection, R{query_function}, timeout, _, false, false, false, bytes_sent, true); } template -array f$typed_rpc_tl_query(const class_instance &connection, - const array> &query_functions, - double timeout = -1.0, - bool ignore_answer = false) { - static_assert(std::is_base_of::value, "Unexpected type"); - static_assert(std::is_same::value, "Unexpected type"); - - array queries(query_functions.size()); +array f$typed_rpc_tl_query(const class_instance &connection, const array> &query_functions, double timeout = -1.0, + bool ignore_answer = false, class_instance requests_extra_info = {}, + bool need_responses_extra_info = false) { + static_assert(std::is_base_of_v, "Unexpected type"); + static_assert(std::is_same_v, "Unexpected type"); + + if (ignore_answer && need_responses_extra_info) { + php_warning( + "Both $ignore_answer and $need_responses_extra_info are 'true'. Can't collect metrics for ignored answers"); + } + size_t bytes_sent = 0; + bool collect_resp_extra_info = !ignore_answer && need_responses_extra_info; + array queries{query_functions.size()}; + array req_extra_info_arr{query_functions.size()}; + for (auto it = query_functions.begin(); it != query_functions.end(); ++it) { - int64_t rpc_query = typed_rpc_tl_query_impl(connection, R{it.get_value()}, timeout, ignore_answer, true, bytes_sent, false); + rpc_request_extra_info_t req_ei{}; + + int64_t rpc_query = typed_rpc_tl_query_impl(connection, R{it.get_value()}, timeout, req_ei, collect_resp_extra_info, ignore_answer, true, bytes_sent, + false); + queries.set_value(it.get_key(), rpc_query); + req_extra_info_arr.set_value(it.get_key(), std::move(req_ei)); } + if (bytes_sent > 0) { f$rpc_flush(); } + if (!requests_extra_info.is_null()) { + requests_extra_info->extra_info_arr_ = std::move(req_extra_info_arr); + } + return queries; } template class_instance f$typed_rpc_tl_query_result_one(I query_id) { - static_assert(std::is_same::value, "Unexpected type"); - static_assert(std::is_same::value, "Unexpected type"); + static_assert(std::is_same_v, "Unexpected type"); + static_assert(std::is_same_v, "Unexpected type"); return typed_rpc_tl_query_result_one_impl(query_id, F::get()); } template array> f$typed_rpc_tl_query_result(const array &query_ids) { - static_assert(std::is_same::value, "Unexpected type"); - static_assert(std::is_same::value, "Unexpected type"); + static_assert(std::is_same_v, "Unexpected type"); + static_assert(std::is_same_v, "Unexpected type"); return typed_rpc_tl_query_result_impl(query_ids, F::get()); } template array> f$typed_rpc_tl_query_result_synchronously(const array &query_ids) { - static_assert(std::is_same::value, "Unexpected type"); - static_assert(std::is_same::value, "Unexpected type"); + static_assert(std::is_same_v, "Unexpected type"); + static_assert(std::is_same_v, "Unexpected type"); return typed_rpc_tl_query_result_synchronously_impl(query_ids, F::get()); } diff --git a/tests/python/tests/rpc/__init__.py b/tests/python/tests/rpc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/python/tests/rpc/php/VK/TL/RpcFunction.php b/tests/python/tests/rpc/php/VK/TL/RpcFunction.php new file mode 100644 index 0000000000..65f39e5528 --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/RpcFunction.php @@ -0,0 +1,33 @@ +binlog_pos !== null) { + $mask |= self::BIT_BINLOG_POS_0; + } + + if ($this->binlog_time !== null) { + $mask |= self::BIT_BINLOG_TIME_1; + } + + if ($this->engine_pid) { + $mask |= self::BIT_ENGINE_PID_2; + } + + if ($this->request_size !== null && $this->response_size !== null) { + $mask |= (self::BIT_REQUEST_SIZE_3 | self::BIT_RESPONSE_SIZE_3); + } + + if ($this->failed_subqueries !== null) { + $mask |= self::BIT_FAILED_SUBQUERIES_4; + } + + if ($this->compression_version !== null) { + $mask |= self::BIT_COMPRESSION_VERSION_5; + } + + if ($this->stats !== null) { + $mask |= self::BIT_STATS_6; + } + + if ($this->epoch_number !== null && $this->view_number !== null) { + $mask |= (self::BIT_EPOCH_NUMBER_27 | self::BIT_VIEW_NUMBER_27); + } + + return $mask; + } + +} diff --git a/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseError.php b/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseError.php new file mode 100644 index 0000000000..bc5375787b --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseError.php @@ -0,0 +1,61 @@ +error_code = $error_code; + $this->error = $error; + } + + /** + * @return null + */ + public function getResult() { + return null; + } + + /** + * @return null + */ + public function getHeader() { + return null; + } + + /** + * @return bool + */ + public function isError() { + return true; + } + + /** + * @return TL\_common\Types\rpcResponseError + */ + public function getError() { + return $this; + } + +} diff --git a/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseHeader.php b/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseHeader.php new file mode 100644 index 0000000000..6c4ea402d7 --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseHeader.php @@ -0,0 +1,66 @@ +flags = $flags; + $this->extra = $extra; + $this->result = $result; + } + + /** + * @return TL\RpcFunctionReturnResult + */ + public function getResult() { + return $this->result; + } + + /** + * @return TL\_common\Types\rpcResponseHeader + */ + public function getHeader() { + return $this; + } + + /** + * @return bool + */ + public function isError() { + return false; + } + + /** + * @return null + */ + public function getError() { + return null; + } + +} diff --git a/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseOk.php b/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseOk.php new file mode 100644 index 0000000000..b08a81c5a6 --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcResponseOk.php @@ -0,0 +1,56 @@ +result = $result; + } + + /** + * @return TL\RpcFunctionReturnResult + */ + public function getResult() { + return $this->result; + } + + /** + * @return null + */ + public function getHeader() { + return null; + } + + /** + * @return bool + */ + public function isError() { + return false; + } + + /** + * @return null + */ + public function getError() { + return null; + } + +} diff --git a/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_pid.php b/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_pid.php new file mode 100644 index 0000000000..4b90256b51 --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_pid.php @@ -0,0 +1,68 @@ +value; + } + warning('Unexpected result type in functionReturnValue: ' . ($function_return_result ? get_class($function_return_result) : 'null')); + return (new engine_pid_result())->value; + } + + /** + * @kphp-inline + * + * @param TL\RpcResponse $response + * @return TL\net\Types\net_pid + */ + public static function result(TL\RpcResponse $response) { + return self::functionReturnValue($response->getResult()); + } + + /** + * @kphp-inline + * + * @return string + */ + public function getTLFunctionName() { + return 'engine.pid'; + } + +} + +/** + * @kphp-tl-class + */ +class engine_pid_result implements TL\RpcFunctionReturnResult { + + /** @var TL\net\Types\net_pid */ + public $value = null; + +} diff --git a/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_sleep.php b/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_sleep.php new file mode 100644 index 0000000000..fae23aacb9 --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_sleep.php @@ -0,0 +1,72 @@ +time_ms = $time_ms; + } + + /** + * @param TL\RpcFunctionReturnResult $function_return_result + * @return boolean + */ + public static function functionReturnValue($function_return_result) { + if ($function_return_result instanceof engine_sleep_result) { + return $function_return_result->value; + } + warning('Unexpected result type in functionReturnValue: ' . ($function_return_result ? get_class($function_return_result) : 'null')); + return (new engine_sleep_result())->value; + } + + /** + * @kphp-inline + * + * @param TL\RpcResponse $response + * @return boolean + */ + public static function result(TL\RpcResponse $response) { + return self::functionReturnValue($response->getResult()); + } + + /** + * @kphp-inline + * + * @return string + */ + public function getTLFunctionName() { + return 'engine.sleep'; + } + +} + +/** + * @kphp-tl-class + */ +class engine_sleep_result implements TL\RpcFunctionReturnResult { + + /** @var boolean */ + public $value = false; + +} diff --git a/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_stat.php b/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_stat.php new file mode 100644 index 0000000000..1cba52826d --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_stat.php @@ -0,0 +1,68 @@ +value; + } + warning('Unexpected result type in functionReturnValue: ' . ($function_return_result ? get_class($function_return_result) : 'null')); + return (new engine_stat_result())->value; + } + + /** + * @kphp-inline + * + * @param TL\RpcResponse $response + * @return string[] + */ + public static function result(TL\RpcResponse $response) { + return self::functionReturnValue($response->getResult()); + } + + /** + * @kphp-inline + * + * @return string + */ + public function getTLFunctionName() { + return 'engine.stat'; + } + +} + +/** + * @kphp-tl-class + */ +class engine_stat_result implements TL\RpcFunctionReturnResult { + + /** @var string[] */ + public $value = []; + +} diff --git a/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_version.php b/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_version.php new file mode 100644 index 0000000000..e628eb315e --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/engine/Functions/engine_version.php @@ -0,0 +1,68 @@ +value; + } + warning('Unexpected result type in functionReturnValue: ' . ($function_return_result ? get_class($function_return_result) : 'null')); + return (new engine_version_result())->value; + } + + /** + * @kphp-inline + * + * @param TL\RpcResponse $response + * @return string + */ + public static function result(TL\RpcResponse $response) { + return self::functionReturnValue($response->getResult()); + } + + /** + * @kphp-inline + * + * @return string + */ + public function getTLFunctionName() { + return 'engine.version'; + } + +} + +/** + * @kphp-tl-class + */ +class engine_version_result implements TL\RpcFunctionReturnResult { + + /** @var string */ + public $value = ''; + +} diff --git a/tests/python/tests/rpc/php/VK/TL/net/Types/net_pid.php b/tests/python/tests/rpc/php/VK/TL/net/Types/net_pid.php new file mode 100644 index 0000000000..3da639dd2b --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/net/Types/net_pid.php @@ -0,0 +1,36 @@ +ip = $ip; + $this->port_pid = $port_pid; + $this->utime = $utime; + } + +} diff --git a/tests/python/tests/rpc/php/index.php b/tests/python/tests/rpc/php/index.php new file mode 100644 index 0000000000..5bde09d781 --- /dev/null +++ b/tests/python/tests/rpc/php/index.php @@ -0,0 +1,88 @@ + "engine.stat"], ['_' => "engine.pid"], ['_' => "engine.version"], ["_" => "engine.sleep", "time_ms" => (int)(200)]]; + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"]); + $requests_extra_info = new \KphpRpcRequestsExtraInfo; + + $query_ids = rpc_tl_query($conn, $queries, -1.0, false, $requests_extra_info, true); + + $res = rpc_tl_query_result($query_ids); + + $responses_extra_info = []; + foreach ($query_ids as $q_id) { + $extra_info = extract_kphp_rpc_response_extra_info($q_id); + if (is_null($extra_info)) { + critical_error("got null rpc response extra info after processing an rpc response!"); + } + array_push($responses_extra_info, $extra_info); + } + + echo json_encode([ + "result" => $res["result"], + "requests_extra_info" => array_map(fn($req_tup): array => [$req_tup[0]], $requests_extra_info->get()), + "responses_extra_info" => array_map(fn($resp_tup): array => [$resp_tup[0], $resp_tup[1]], $responses_extra_info), + ]); +} + +function get_kphp_typed_rpc_extra_info() { + $queries = [new engine_stat(), new engine_pid(), new engine_version(), new engine_sleep(200)]; + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"]); + $requests_extra_info = new \KphpRpcRequestsExtraInfo; + + $query_ids = typed_rpc_tl_query($conn, $queries, -1.0, false, $requests_extra_info, true); + + $res = typed_rpc_tl_query_result($query_ids); + + $responses_extra_info = []; + foreach ($query_ids as $q_id) { + $extra_info = extract_kphp_rpc_response_extra_info($q_id); + if (is_null($extra_info)) { + critical_error("got null rpc response extra info after processing an rpc response!"); + } + array_push($responses_extra_info, $extra_info); + } + + foreach ($res as $resp) { + if ($resp->isError()) { + critical_error("bad rpc response"); + } + } + + echo json_encode([ + "requests_extra_info" => array_map(fn($req_tup): array => [$req_tup[0]], $requests_extra_info->get()), + "responses_extra_info" => array_map(fn($resp_tup): array => [$resp_tup[0], $resp_tup[1]], $responses_extra_info), + ]); +} + +function do_http_worker() { + switch($_SERVER["PHP_SELF"]) { + case "/get_kphp_untyped_rpc_extra_info": { + get_kphp_untyped_rpc_extra_info(); + return; + } + case "/get_kphp_typed_rpc_extra_info": { + get_kphp_typed_rpc_extra_info(); + return; + } + } + + critical_error("unreachable code!"); +} + +do_http_worker(); diff --git a/tests/python/tests/rpc/test_rpc_extra_info.py b/tests/python/tests/rpc/test_rpc_extra_info.py new file mode 100644 index 0000000000..d2ddb27611 --- /dev/null +++ b/tests/python/tests/rpc/test_rpc_extra_info.py @@ -0,0 +1,57 @@ +import json + +from python.lib.testcase import KphpServerAutoTestCase + + +class TestRpcExtraInfo(KphpServerAutoTestCase): + def test_untyped_rpc_extra_info(self): + rpc_extra_info = self.kphp_server.http_get( + "/get_kphp_untyped_rpc_extra_info?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_extra_info.status_code, 200) + self.assertNotEqual(rpc_extra_info.text, "") + + output = json.loads(rpc_extra_info.text) + + self.assertNotEqual(output["result"], "") + + req_extra_info_arr = output["requests_extra_info"] + + requests_size_arr = [28, 28, 28, 32] + self.assertEqual(len(req_extra_info_arr), 4) + for i, extra_info in enumerate(req_extra_info_arr): + self.assertEqual(extra_info[0], requests_size_arr[i]) + + resp_extra_info_arr = output["responses_extra_info"] + self.assertEqual(len(resp_extra_info_arr), 4) + for extra_info in resp_extra_info_arr: + self.assertNotEqual(extra_info[0], 0) + self.assertNotEqual(extra_info[1], 0) + + # check engine response time using extra info of engine.sleep request + self.assertTrue(resp_extra_info_arr[-1][1] > 0.2) + + def test_typed_rpc_extra_info(self): + rpc_extra_info = self.kphp_server.http_get( + "/get_kphp_typed_rpc_extra_info?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_extra_info.status_code, 200) + self.assertNotEqual(rpc_extra_info.text, "") + + output = json.loads(rpc_extra_info.text) + + req_extra_info_arr = output["requests_extra_info"] + + requests_size_arr = [28, 28, 28, 32] + self.assertEqual(len(req_extra_info_arr), 4) + for i, extra_info in enumerate(req_extra_info_arr): + self.assertEqual(extra_info[0], requests_size_arr[i]) + + resp_extra_info_arr = output["responses_extra_info"] + self.assertEqual(len(resp_extra_info_arr), 4) + for extra_info in resp_extra_info_arr: + self.assertNotEqual(extra_info[0], 0) + self.assertNotEqual(extra_info[1], 0) + + # check engine response time using extra info of engine.sleep request + self.assertTrue(resp_extra_info_arr[-1][1] > 0.2) diff --git a/vkext/vkext-rpc-tl-serialization.cpp b/vkext/vkext-rpc-tl-serialization.cpp index 06325e0cb4..5d7e152dcb 100644 --- a/vkext/vkext-rpc-tl-serialization.cpp +++ b/vkext/vkext-rpc-tl-serialization.cpp @@ -1176,14 +1176,14 @@ void vk_rpc_tl_query(INTERNAL_FUNCTION_PARAMETERS) { ADD_CNT (parse); START_TIMER (parse); int argc = ZEND_NUM_ARGS (); - VK_ZVAL_API_ARRAY z[5]; + VK_ZVAL_API_ARRAY z[6]; if (argc < 2) { vkext_reset_error(); vkext_error(VKEXT_ERROR_INVALID_CALL, "Not enough parameters for rpc_tl_query"); END_TIMER (parse); RETURN_EMPTY_ARRAY(); } - if (zend_get_parameters_array_ex (argc > 4 ? 4 : argc, z) == FAILURE) { + if (zend_get_parameters_array_ex (argc > 6 ? 6 : argc, z) == FAILURE) { vkext_reset_error(); vkext_error(VKEXT_ERROR_INVALID_CALL, "Can't parse parameters for rpc_tl_query"); END_TIMER (parse); diff --git a/vkext/vkext.stub.php b/vkext/vkext.stub.php index 291c9a462f..e603ee1d4e 100644 --- a/vkext/vkext.stub.php +++ b/vkext/vkext.stub.php @@ -221,7 +221,9 @@ function rpc_tl_pending_queries_count() : int { return 0; } -function rpc_tl_query(int $rpc_connection, array $tl_queries, float $timeout = -1.0, bool $ignore_answer = false) : array { +function rpc_tl_query(int $rpc_connection, array $tl_queries, float $timeout = -1.0, + bool $ignore_answer = false, \KphpRpcRequestsExtraInfo $requests_extra_info = null, + bool $need_responses_extra_info = false) : array { return []; } @@ -246,7 +248,9 @@ function rpc_tl_query_result_one(int $request_id, float $timeout = -1.0) { return []; } -function typed_rpc_tl_query(int $rpc_connection, array $tl_queries, float $timeout = -1.0, bool $ignore_answer = false) : array { +function typed_rpc_tl_query(int $rpc_connection, array $tl_queries, float $timeout = -1.0, + bool $ignore_answer = false, \KphpRpcRequestsExtraInfo $requests_extra_info = null, + bool $need_responses_extra_info = false) : array { return []; } diff --git a/vkext/vkext_arginfo.h b/vkext/vkext_arginfo.h index 1adb84e722..ebdd1de87a 100644 --- a/vkext/vkext_arginfo.h +++ b/vkext/vkext_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 483315f5e4ee09b477d536fe066baf385afd6afb */ + * Stub hash: f4071c62e1d7182568496ff05daba48416356127 */ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_vk_hello_world, 0, 0, IS_STRING, 0) ZEND_END_ARG_INFO() @@ -155,6 +155,8 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_rpc_tl_query, 0, 2, IS_ARRAY, 0) ZEND_ARG_TYPE_INFO(0, tl_queries, IS_ARRAY, 0) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, timeout, IS_DOUBLE, 0, "-1.0") ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, ignore_answer, _IS_BOOL, 0, "false") + ZEND_ARG_OBJ_INFO_WITH_DEFAULT_VALUE(0, requests_extra_info, KphpRpcRequestsExtraInfo, 0, "null") + ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, need_responses_extra_info, _IS_BOOL, 0, "false") ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_INFO_EX(arginfo_rpc_tl_query_one, 0, 0, 2) diff --git a/vkext/vkext_legacy_arginfo.h b/vkext/vkext_legacy_arginfo.h index 53784aa9c9..fc5e530127 100644 --- a/vkext/vkext_legacy_arginfo.h +++ b/vkext/vkext_legacy_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 483315f5e4ee09b477d536fe066baf385afd6afb */ + * Stub hash: f4071c62e1d7182568496ff05daba48416356127 */ ZEND_BEGIN_ARG_INFO_EX(arginfo_vk_hello_world, 0, 0, 0) ZEND_END_ARG_INFO() @@ -147,6 +147,8 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_rpc_tl_query, 0, 0, 2) ZEND_ARG_INFO(0, tl_queries) ZEND_ARG_INFO(0, timeout) ZEND_ARG_INFO(0, ignore_answer) + ZEND_ARG_INFO(0, requests_extra_info) + ZEND_ARG_INFO(0, need_responses_extra_info) ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_INFO_EX(arginfo_rpc_tl_query_one, 0, 0, 2) From 3d1f228ffdd987c298092baefa6de2ed3978ab50 Mon Sep 17 00:00:00 2001 From: Alexander Polyakov Date: Sat, 27 Apr 2024 13:28:43 +0300 Subject: [PATCH 14/89] Fix SIGSEGV in openssl_random_pseudo_bytes and xor_strings (#985) SIGSEGV was caused by using wrong string's constructor. The constructor we used was an optimization for single-character strings. Such strings are stored in `.rodata`. Writing to these strings' buffer led to a SIGSEGV --- runtime/openssl.cpp | 4 +-- runtime/string_decl.inl | 4 +++ runtime/string_functions.cpp | 2 +- .../openssl/9_openssl_random_pseudo_bytes.php | 33 +++++++++++++++++++ .../phpt/string_functions/010_xor_strings.php | 24 ++++++++++++++ 5 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 tests/phpt/openssl/9_openssl_random_pseudo_bytes.php create mode 100644 tests/phpt/string_functions/010_xor_strings.php diff --git a/runtime/openssl.cpp b/runtime/openssl.cpp index a85ce4c62f..cf7c91433b 100644 --- a/runtime/openssl.cpp +++ b/runtime/openssl.cpp @@ -601,7 +601,7 @@ Optional f$openssl_random_pseudo_bytes(int64_t length) { if (length <= 0 || length > string::max_size()) { return false; } - string buffer(static_cast(length), ' '); + string buffer{static_cast(length), false}; timeval tv{}; gettimeofday(&tv, nullptr); @@ -610,7 +610,7 @@ Optional f$openssl_random_pseudo_bytes(int64_t length) { if (RAND_bytes(reinterpret_cast(buffer.buffer()), static_cast(length)) <= 0) { return false; } - return std::move(buffer); + return buffer; } diff --git a/runtime/string_decl.inl b/runtime/string_decl.inl index 950f1cab41..1d5132ab3e 100644 --- a/runtime/string_decl.inl +++ b/runtime/string_decl.inl @@ -74,6 +74,8 @@ private: inline void set_size(size_type new_size); inline static char *create(const char *beg, const char *end); + // IMPORTANT: this function may return read-only strings for n == 0 and n == 1. + // Use it unless you have to manually write something into the buffer. inline static char *create(size_type req, char c); inline static char *create(size_type req, bool b); @@ -97,6 +99,8 @@ public: inline string(string &&str) noexcept; inline string(const char *s, size_type n); inline explicit string(const char *s); + // IMPORTANT: this constructor may return read-only strings for n == 0 and n == 1. + // Use it unless you have to manually operate with string's internal buffer. inline string(size_type n, char c); inline string(size_type n, bool b); inline explicit string(int64_t i); diff --git a/runtime/string_functions.cpp b/runtime/string_functions.cpp index f5eb182881..a467e36d29 100644 --- a/runtime/string_functions.cpp +++ b/runtime/string_functions.cpp @@ -2844,7 +2844,7 @@ string f$wordwrap(const string &str, int64_t width, const string &brk, bool cut) string f$xor_strings(const string &s, const string &t) { string::size_type length = min(s.size(), t.size()); - string result(length, ' '); + string result{length, false}; const char *s_str = s.c_str(); const char *t_str = t.c_str(); char *res_str = result.buffer(); diff --git a/tests/phpt/openssl/9_openssl_random_pseudo_bytes.php b/tests/phpt/openssl/9_openssl_random_pseudo_bytes.php new file mode 100644 index 0000000000..26b1a0cc36 --- /dev/null +++ b/tests/phpt/openssl/9_openssl_random_pseudo_bytes.php @@ -0,0 +1,33 @@ +@ok + Date: Fri, 3 May 2024 20:33:08 +0300 Subject: [PATCH 15/89] Fix RPC queries being doubly wrapped (#987) In some cases RPC queries that are wrapped with RpcDestFlags, RpcDestActor, or RpcDestActorFlags are wrapped one more time with RpcDestActorFlags. This PR fixes this behaviour. --- common/rpc-headers.cpp | 103 ++++++++-- common/rpc-headers.h | 19 +- runtime/rpc.cpp | 45 +++-- runtime/typed_rpc.h | 3 +- .../VK/TL/_common/Functions/rpcDestActor.php | 77 +++++++ .../_common/Functions/rpcDestActorFlags.php | 87 ++++++++ .../VK/TL/_common/Functions/rpcDestFlags.php | 82 ++++++++ .../VK/TL/_common/Types/rpcInvokeReqExtra.php | 191 ++++++++++++++++++ tests/python/tests/rpc/php/index.php | 116 +++++------ tests/python/tests/rpc/php/rpc_extra_info.php | 71 +++++++ tests/python/tests/rpc/php/rpc_wrappers.php | 161 +++++++++++++++ tests/python/tests/rpc/test_rpc_extra_info.py | 4 +- tests/python/tests/rpc/test_rpc_wrappers.py | 109 ++++++++++ vkext/vkext-rpc.cpp | 20 +- 14 files changed, 974 insertions(+), 114 deletions(-) create mode 100644 tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestActor.php create mode 100644 tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestActorFlags.php create mode 100644 tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestFlags.php create mode 100644 tests/python/tests/rpc/php/VK/TL/_common/Types/rpcInvokeReqExtra.php create mode 100644 tests/python/tests/rpc/php/rpc_extra_info.php create mode 100644 tests/python/tests/rpc/php/rpc_wrappers.php create mode 100644 tests/python/tests/rpc/test_rpc_wrappers.py diff --git a/common/rpc-headers.cpp b/common/rpc-headers.cpp index 9e28d740b3..9548dd3297 100644 --- a/common/rpc-headers.cpp +++ b/common/rpc-headers.cpp @@ -2,28 +2,91 @@ // Copyright (c) 2020 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "common/algorithms/find.h" #include "common/rpc-headers.h" +#include "common/algorithms/find.h" #include "common/tl/constants/common.h" -size_t fill_extra_headers_if_needed(RpcExtraHeaders &extra_headers, uint32_t function_magic, int actor_id, bool ignore_answer) { - size_t extra_headers_size = 0; - bool need_actor = actor_id != 0 && vk::none_of_equal(function_magic, TL_RPC_DEST_ACTOR, TL_RPC_DEST_ACTOR_FLAGS); - bool need_flags = ignore_answer && vk::none_of_equal(function_magic, TL_RPC_DEST_FLAGS, TL_RPC_DEST_ACTOR_FLAGS); - - if (need_actor && need_flags) { - extra_headers.rpc_dest_actor_flags.op = TL_RPC_DEST_ACTOR_FLAGS; - extra_headers.rpc_dest_actor_flags.actor_id = actor_id; - extra_headers.rpc_dest_actor_flags.flags = vk::tl::common::rpc_invoke_req_extra_flags::no_result; - extra_headers_size = sizeof(extra_headers.rpc_dest_actor_flags); - } else if (need_actor) { - extra_headers.rpc_dest_actor.op = TL_RPC_DEST_ACTOR; - extra_headers.rpc_dest_actor.actor_id = actor_id; - extra_headers_size = sizeof(extra_headers.rpc_dest_actor); - } else if (need_flags) { - extra_headers.rpc_dest_flags.op = TL_RPC_DEST_FLAGS; - extra_headers.rpc_dest_flags.flags = vk::tl::common::rpc_invoke_req_extra_flags::no_result; - extra_headers_size = sizeof(extra_headers.rpc_dest_flags); + +RegularizeWrappersReturnT regularize_wrappers(const char *rpc_payload, std::int32_t actor_id, bool ignore_result) { + static_assert(sizeof(RpcDestActorFlagsHeaders) >= sizeof(RpcDestActorHeaders)); + static_assert(sizeof(RpcDestActorFlagsHeaders) >= sizeof(RpcDestFlagsHeaders)); + + const auto cur_wrapper{*reinterpret_cast(rpc_payload)}; + const auto function_magic{*reinterpret_cast(rpc_payload)}; + + if (actor_id == 0 && !ignore_result && vk::none_of_equal(function_magic, TL_RPC_DEST_ACTOR, TL_RPC_DEST_FLAGS, TL_RPC_DEST_ACTOR_FLAGS)) { + return {std::nullopt, 0, std::nullopt, nullptr}; + } + + RpcExtraHeaders extra_headers{}; + const std::size_t new_wrapper_size{sizeof(RpcDestActorFlagsHeaders)}; + std::size_t cur_wrapper_size{0}; + std::int32_t cur_wrapper_actor_id{0}; + bool cur_wrapper_ignore_result{false}; + + switch (function_magic) { + case TL_RPC_DEST_ACTOR_FLAGS: + cur_wrapper_size = sizeof(RpcDestActorFlagsHeaders); + cur_wrapper_actor_id = cur_wrapper.rpc_dest_actor_flags.actor_id; + cur_wrapper_ignore_result = static_cast(cur_wrapper.rpc_dest_actor_flags.flags & vk::tl::common::rpc_invoke_req_extra_flags::no_result); + + extra_headers.rpc_dest_actor_flags.op = TL_RPC_DEST_ACTOR_FLAGS; + extra_headers.rpc_dest_actor_flags.actor_id = actor_id != 0 ? actor_id : cur_wrapper.rpc_dest_actor_flags.actor_id; + if (ignore_result) { + extra_headers.rpc_dest_actor_flags.flags = cur_wrapper.rpc_dest_actor_flags.flags | vk::tl::common::rpc_invoke_req_extra_flags::no_result; + } else { + extra_headers.rpc_dest_actor_flags.flags = cur_wrapper.rpc_dest_actor_flags.flags & ~vk::tl::common::rpc_invoke_req_extra_flags::no_result; + } + + break; + case TL_RPC_DEST_ACTOR: + cur_wrapper_size = sizeof(RpcDestActorHeaders); + cur_wrapper_actor_id = cur_wrapper.rpc_dest_actor.actor_id; + + extra_headers.rpc_dest_actor_flags.op = TL_RPC_DEST_ACTOR_FLAGS; + extra_headers.rpc_dest_actor_flags.actor_id = actor_id != 0 ? actor_id : cur_wrapper.rpc_dest_actor.actor_id; + extra_headers.rpc_dest_actor_flags.flags = ignore_result ? vk::tl::common::rpc_invoke_req_extra_flags::no_result : 0x0; + + break; + case TL_RPC_DEST_FLAGS: + cur_wrapper_size = sizeof(RpcDestFlagsHeaders); + cur_wrapper_ignore_result = static_cast(cur_wrapper.rpc_dest_flags.flags & vk::tl::common::rpc_invoke_req_extra_flags::no_result); + + extra_headers.rpc_dest_actor_flags.op = TL_RPC_DEST_ACTOR_FLAGS; + extra_headers.rpc_dest_actor_flags.actor_id = actor_id; + if (ignore_result) { + extra_headers.rpc_dest_actor_flags.flags = cur_wrapper.rpc_dest_flags.flags | vk::tl::common::rpc_invoke_req_extra_flags::no_result; + } else { + extra_headers.rpc_dest_actor_flags.flags = cur_wrapper.rpc_dest_flags.flags & ~vk::tl::common::rpc_invoke_req_extra_flags::no_result; + } + + break; + default: + // we don't have a cur_wrapper, but we do have 'actor_id' or 'ignore_result' set + extra_headers.rpc_dest_actor_flags.op = TL_RPC_DEST_ACTOR_FLAGS; + extra_headers.rpc_dest_actor_flags.actor_id = actor_id; + extra_headers.rpc_dest_actor_flags.flags = ignore_result ? vk::tl::common::rpc_invoke_req_extra_flags::no_result : 0x0; + + break; } - return extra_headers_size; + + decltype(RegularizeWrappersReturnT{}.opt_actor_id_warning_info) opt_actor_id_warning{}; + if (actor_id != 0 && cur_wrapper_actor_id != 0) { + opt_actor_id_warning.emplace("inaccurate use of 'actor_id': '%d' was passed into RPC connection constructor, " + "but '%d' was already set in RpcDestActor or RpcDestActorFlags\n", + actor_id, cur_wrapper_actor_id); + } + + const char *opt_ignore_result_warning_msg{nullptr}; + if (!ignore_result && cur_wrapper_ignore_result) { + opt_ignore_result_warning_msg = "inaccurate use of 'ignore_answer': 'false' was passed into TL query function (e.g., rpc_tl_query), " + "but 'true' was already set in RpcDestFlags or RpcDestActorFlags\n"; + } + + return { + std::pair{extra_headers, new_wrapper_size}, + cur_wrapper_size, + std::move(opt_actor_id_warning), + opt_ignore_result_warning_msg, + }; } diff --git a/common/rpc-headers.h b/common/rpc-headers.h index 355ae2ed99..5dceecc233 100644 --- a/common/rpc-headers.h +++ b/common/rpc-headers.h @@ -4,8 +4,10 @@ #pragma once -#include #include +#include +#include +#include #pragma pack(push, 1) @@ -44,4 +46,17 @@ struct RpcHeaders { #pragma pack(pop) -size_t fill_extra_headers_if_needed(RpcExtraHeaders &extra_headers, uint32_t function_magic, int actor_id, bool ignore_answer); +struct RegularizeWrappersReturnT { + /// Optionally contains a new wrapper and its size + std::optional> opt_new_wrapper; + /// The size of a wrapper found in rpc payload (0 if there is no one) + std::size_t cur_wrapper_size; + /// Optionally contains a tuple of . + /// If not std::nullopt, can be used to warn about actor_id redefinition, for example, + /// 'php_warning(format_str, current_wrapper_actor_id, new_actor_id)' + std::optional> opt_actor_id_warning_info; + /// Optionally contains a string. If not nullptr, can be used to warn about inaccurate usage of 'ignore_result'. + const char *opt_ignore_result_warning_msg; +}; + +RegularizeWrappersReturnT regularize_wrappers(const char *rpc_payload, std::int32_t actor_id, bool ignore_result); diff --git a/runtime/rpc.cpp b/runtime/rpc.cpp index 27e7188865..d476cc54e0 100644 --- a/runtime/rpc.cpp +++ b/runtime/rpc.cpp @@ -5,6 +5,7 @@ #include "runtime/rpc.h" #include +#include #include #include "common/rpc-error-codes.h" @@ -679,22 +680,40 @@ int64_t rpc_send_impl(const class_instance &conn, double timeou store_int(-1); // reserve for crc32 php_assert (data_buf.size() % sizeof(int) == 0); - const char *rpc_payload_start = data_buf.c_str() + sizeof(RpcHeaders); - size_t rpc_payload_size = data_buf.size() - sizeof(RpcHeaders); - uint32_t function_magic = CurrentProcessingQuery::get().get_last_stored_tl_function_magic(); - RpcExtraHeaders extra_headers{}; - size_t extra_headers_size = fill_extra_headers_if_needed(extra_headers, function_magic, conn.get()->actor_id, ignore_answer); + const auto [opt_new_wrapper, cur_wrapper_size, opt_actor_id_warning_info, opt_ignore_result_warning_msg]{ + regularize_wrappers(data_buf.c_str() + sizeof(RpcHeaders), conn.get()->actor_id, ignore_answer)}; - const auto request_size = static_cast(data_buf.size() + extra_headers_size); - char *p = static_cast(dl::allocate(request_size)); + if (opt_actor_id_warning_info.has_value()) { + const auto [msg, cur_wrapper_actor_id, new_wrapper_actor_id]{opt_actor_id_warning_info.value()}; + php_warning(msg, cur_wrapper_actor_id, new_wrapper_actor_id); + } + if (opt_ignore_result_warning_msg != nullptr) { + php_warning("%s", opt_ignore_result_warning_msg); + } + + char *request_buf{nullptr}; + std::size_t request_size{0}; - // Memory will look like this: + // 'request_buf' will look like this: // [ RpcHeaders (reserved in f$rpc_clean) ] [ RpcExtraHeaders (optional) ] [ payload ] - memcpy(p, data_buf.c_str(), sizeof(RpcHeaders)); - memcpy(p + sizeof(RpcHeaders), &extra_headers, extra_headers_size); - memcpy(p + sizeof(RpcHeaders) + extra_headers_size, rpc_payload_start, rpc_payload_size); + if (opt_new_wrapper.has_value()) { + const auto [new_wrapper, new_wrapper_size]{opt_new_wrapper.value()}; + request_size = data_buf.size() - cur_wrapper_size + new_wrapper_size; + request_buf = static_cast(dl::allocate(request_size)); + + std::memcpy(request_buf, data_buf.c_str(), sizeof(RpcHeaders)); + std::memcpy(request_buf + sizeof(RpcHeaders), &new_wrapper, new_wrapper_size); + std::memcpy(request_buf + sizeof(RpcHeaders) + new_wrapper_size, + data_buf.c_str() + sizeof(RpcHeaders) + cur_wrapper_size, + data_buf.size() - sizeof(RpcHeaders) - cur_wrapper_size); + } else { + request_size = data_buf.size(); + request_buf = static_cast(dl::allocate(request_size)); + + std::memcpy(request_buf, data_buf.c_str(), request_size); + } - slot_id_t q_id = rpc_send_query(conn.get()->host_num, p, static_cast(request_size), timeout_convert_to_ms(timeout)); + slot_id_t q_id = rpc_send_query(conn.get()->host_num, request_buf, static_cast(request_size), timeout_convert_to_ms(timeout)); // request's statistics req_extra_info = rpc_request_extra_info_t{request_size}; @@ -739,7 +758,7 @@ int64_t rpc_send_impl(const class_instance &conn, double timeou double send_timestamp = std::chrono::duration{std::chrono::system_clock::now().time_since_epoch()}.count(); cur->resumable_id = register_forked_resumable(new rpc_resumable(q_id)); - cur->function_magic = function_magic; + cur->function_magic = CurrentProcessingQuery::get().get_last_stored_tl_function_magic(); cur->actor_or_port = conn.get()->actor_id > 0 ? conn.get()->actor_id : -conn.get()->port; cur->timer = nullptr; diff --git a/runtime/typed_rpc.h b/runtime/typed_rpc.h index f756084a51..6b57fdf5a1 100644 --- a/runtime/typed_rpc.h +++ b/runtime/typed_rpc.h @@ -49,8 +49,7 @@ array f$typed_rpc_tl_query(const class_instance &conne static_assert(std::is_same_v, "Unexpected type"); if (ignore_answer && need_responses_extra_info) { - php_warning( - "Both $ignore_answer and $need_responses_extra_info are 'true'. Can't collect metrics for ignored answers"); + php_warning("Both $ignore_answer and $need_responses_extra_info are 'true'. Can't collect metrics for ignored answers"); } size_t bytes_sent = 0; diff --git a/tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestActor.php b/tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestActor.php new file mode 100644 index 0000000000..9b89786da3 --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestActor.php @@ -0,0 +1,77 @@ +actor_id = $actor_id; + $this->query = $query; + } + + /** + * @param TL\RpcFunctionReturnResult $function_return_result + * @return TL\RpcFunctionReturnResult + */ + public static function functionReturnValue($function_return_result) { + if ($function_return_result instanceof rpcDestActor_result) { + return $function_return_result->value; + } + warning('Unexpected result type in functionReturnValue: ' . ($function_return_result ? get_class($function_return_result) : 'null')); + return (new rpcDestActor_result())->value; + } + + /** + * @kphp-inline + * + * @param TL\RpcResponse $response + * @return TL\RpcFunctionReturnResult + */ + public static function result(TL\RpcResponse $response) { + return self::functionReturnValue($response->getResult()); + } + + /** + * @kphp-inline + * + * @return string + */ + public function getTLFunctionName() { + return 'rpcDestActor'; + } + +} + +/** + * @kphp-tl-class + */ +class rpcDestActor_result implements TL\RpcFunctionReturnResult { + + /** @var TL\RpcFunctionReturnResult */ + public $value = null; + +} diff --git a/tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestActorFlags.php b/tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestActorFlags.php new file mode 100644 index 0000000000..9a1b39cec8 --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestActorFlags.php @@ -0,0 +1,87 @@ +actor_id = $actor_id; + $this->flags = $flags; + $this->extra = $extra; + $this->query = $query; + } + + /** + * @param TL\RpcFunctionReturnResult $function_return_result + * @return TL\RpcFunctionReturnResult + */ + public static function functionReturnValue($function_return_result) { + if ($function_return_result instanceof rpcDestActorFlags_result) { + return $function_return_result->value; + } + warning('Unexpected result type in functionReturnValue: ' . ($function_return_result ? get_class($function_return_result) : 'null')); + return (new rpcDestActorFlags_result())->value; + } + + /** + * @kphp-inline + * + * @param TL\RpcResponse $response + * @return TL\RpcFunctionReturnResult + */ + public static function result(TL\RpcResponse $response) { + return self::functionReturnValue($response->getResult()); + } + + /** + * @kphp-inline + * + * @return string + */ + public function getTLFunctionName() { + return 'rpcDestActorFlags'; + } + +} + +/** + * @kphp-tl-class + */ +class rpcDestActorFlags_result implements TL\RpcFunctionReturnResult { + + /** @var TL\RpcFunctionReturnResult */ + public $value = null; + +} diff --git a/tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestFlags.php b/tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestFlags.php new file mode 100644 index 0000000000..c0cdcdd3c3 --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/_common/Functions/rpcDestFlags.php @@ -0,0 +1,82 @@ +flags = $flags; + $this->extra = $extra; + $this->query = $query; + } + + /** + * @param TL\RpcFunctionReturnResult $function_return_result + * @return TL\RpcFunctionReturnResult + */ + public static function functionReturnValue($function_return_result) { + if ($function_return_result instanceof rpcDestFlags_result) { + return $function_return_result->value; + } + warning('Unexpected result type in functionReturnValue: ' . ($function_return_result ? get_class($function_return_result) : 'null')); + return (new rpcDestFlags_result())->value; + } + + /** + * @kphp-inline + * + * @param TL\RpcResponse $response + * @return TL\RpcFunctionReturnResult + */ + public static function result(TL\RpcResponse $response) { + return self::functionReturnValue($response->getResult()); + } + + /** + * @kphp-inline + * + * @return string + */ + public function getTLFunctionName() { + return 'rpcDestFlags'; + } + +} + +/** + * @kphp-tl-class + */ +class rpcDestFlags_result implements TL\RpcFunctionReturnResult { + + /** @var TL\RpcFunctionReturnResult */ + public $value = null; + +} diff --git a/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcInvokeReqExtra.php b/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcInvokeReqExtra.php new file mode 100644 index 0000000000..018cdfa4d3 --- /dev/null +++ b/tests/python/tests/rpc/php/VK/TL/_common/Types/rpcInvokeReqExtra.php @@ -0,0 +1,191 @@ +return_binlog_pos) { + $mask |= self::BIT_RETURN_BINLOG_POS_0; + } + + if ($this->return_binlog_time) { + $mask |= self::BIT_RETURN_BINLOG_TIME_1; + } + + if ($this->return_pid) { + $mask |= self::BIT_RETURN_PID_2; + } + + if ($this->return_request_sizes) { + $mask |= self::BIT_RETURN_REQUEST_SIZES_3; + } + + if ($this->return_failed_subqueries) { + $mask |= self::BIT_RETURN_FAILED_SUBQUERIES_4; + } + + if ($this->return_query_stats) { + $mask |= self::BIT_RETURN_QUERY_STATS_6; + } + + if ($this->no_result) { + $mask |= self::BIT_NO_RESULT_7; + } + + if ($this->wait_binlog_pos !== null) { + $mask |= self::BIT_WAIT_BINLOG_POS_16; + } + + if ($this->string_forward_keys !== null) { + $mask |= self::BIT_STRING_FORWARD_KEYS_18; + } + + if ($this->int_forward_keys !== null) { + $mask |= self::BIT_INT_FORWARD_KEYS_19; + } + + if ($this->string_forward !== null) { + $mask |= self::BIT_STRING_FORWARD_20; + } + + if ($this->int_forward !== null) { + $mask |= self::BIT_INT_FORWARD_21; + } + + if ($this->custom_timeout_ms !== null) { + $mask |= self::BIT_CUSTOM_TIMEOUT_MS_23; + } + + if ($this->supported_compression_version !== null) { + $mask |= self::BIT_SUPPORTED_COMPRESSION_VERSION_25; + } + + if ($this->random_delay !== null) { + $mask |= self::BIT_RANDOM_DELAY_26; + } + + if ($this->return_view_number) { + $mask |= self::BIT_RETURN_VIEW_NUMBER_27; + } + + return $mask; + } + +} diff --git a/tests/python/tests/rpc/php/index.php b/tests/python/tests/rpc/php/index.php index 5bde09d781..c3004102f0 100644 --- a/tests/python/tests/rpc/php/index.php +++ b/tests/python/tests/rpc/php/index.php @@ -1,83 +1,57 @@ "engine.stat"], ['_' => "engine.pid"], ['_' => "engine.version"], ["_" => "engine.sleep", "time_ms" => (int)(200)]]; - $conn = new_rpc_connection("localhost", (int)$_GET["master-port"]); - $requests_extra_info = new \KphpRpcRequestsExtraInfo; - - $query_ids = rpc_tl_query($conn, $queries, -1.0, false, $requests_extra_info, true); +require_once "rpc_extra_info.php"; +require_once "rpc_wrappers.php"; - $res = rpc_tl_query_result($query_ids); - - $responses_extra_info = []; - foreach ($query_ids as $q_id) { - $extra_info = extract_kphp_rpc_response_extra_info($q_id); - if (is_null($extra_info)) { - critical_error("got null rpc response extra info after processing an rpc response!"); +function do_http_worker() { + switch($_SERVER["PHP_SELF"]) { + case "/test_kphp_untyped_rpc_extra_info": { + test_kphp_untyped_rpc_extra_info(); + return; } - array_push($responses_extra_info, $extra_info); - } - - echo json_encode([ - "result" => $res["result"], - "requests_extra_info" => array_map(fn($req_tup): array => [$req_tup[0]], $requests_extra_info->get()), - "responses_extra_info" => array_map(fn($resp_tup): array => [$resp_tup[0], $resp_tup[1]], $responses_extra_info), - ]); -} - -function get_kphp_typed_rpc_extra_info() { - $queries = [new engine_stat(), new engine_pid(), new engine_version(), new engine_sleep(200)]; - $conn = new_rpc_connection("localhost", (int)$_GET["master-port"]); - $requests_extra_info = new \KphpRpcRequestsExtraInfo; - - $query_ids = typed_rpc_tl_query($conn, $queries, -1.0, false, $requests_extra_info, true); - - $res = typed_rpc_tl_query_result($query_ids); - - $responses_extra_info = []; - foreach ($query_ids as $q_id) { - $extra_info = extract_kphp_rpc_response_extra_info($q_id); - if (is_null($extra_info)) { - critical_error("got null rpc response extra info after processing an rpc response!"); + case "/test_kphp_typed_rpc_extra_info": { + test_kphp_typed_rpc_extra_info(); + return; } - array_push($responses_extra_info, $extra_info); - } - - foreach ($res as $resp) { - if ($resp->isError()) { - critical_error("bad rpc response"); + case "/test_rpc_no_wrappers_with_actor_id": { + test_rpc_no_wrappers_with_actor_id(); + return; } - } - - echo json_encode([ - "requests_extra_info" => array_map(fn($req_tup): array => [$req_tup[0]], $requests_extra_info->get()), - "responses_extra_info" => array_map(fn($resp_tup): array => [$resp_tup[0], $resp_tup[1]], $responses_extra_info), - ]); -} - -function do_http_worker() { - switch($_SERVER["PHP_SELF"]) { - case "/get_kphp_untyped_rpc_extra_info": { - get_kphp_untyped_rpc_extra_info(); + case "/test_rpc_no_wrappers_with_ignore_answer": { + test_rpc_no_wrappers_with_ignore_answer(); + return; + } + case "/test_rpc_dest_actor_with_actor_id": { + test_rpc_dest_actor_with_actor_id(); + return; + } + case "/test_rpc_dest_actor_with_ignore_answer": { + test_rpc_dest_actor_with_ignore_answer(); + return; + } + case "/test_rpc_dest_flags_with_actor_id": { + test_rpc_dest_flags_with_actor_id(); + return; + } + case "/test_rpc_dest_flags_with_ignore_answer": { + test_rpc_dest_flags_with_ignore_answer(); + return; + } + case "/test_rpc_dest_flags_with_ignore_answer_1": { + test_rpc_dest_flags_with_ignore_answer_1(); + return; + } + case "/test_rpc_dest_actor_flags_with_actor_id": { + test_rpc_dest_actor_flags_with_actor_id(); + return; + } + case "/test_rpc_dest_actor_flags_with_ignore_answer": { + test_rpc_dest_actor_flags_with_ignore_answer(); return; } - case "/get_kphp_typed_rpc_extra_info": { - get_kphp_typed_rpc_extra_info(); + case "/test_rpc_dest_actor_flags_with_ignore_answer_1": { + test_rpc_dest_actor_flags_with_ignore_answer_1(); return; } } diff --git a/tests/python/tests/rpc/php/rpc_extra_info.php b/tests/python/tests/rpc/php/rpc_extra_info.php new file mode 100644 index 0000000000..f671c3538e --- /dev/null +++ b/tests/python/tests/rpc/php/rpc_extra_info.php @@ -0,0 +1,71 @@ + "engine.stat"], ['_' => "engine.pid"], ['_' => "engine.version"], ["_" => "engine.sleep", "time_ms" => (int)(200)]]; + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"]); + $requests_extra_info = new \KphpRpcRequestsExtraInfo; + + $query_ids = rpc_tl_query($conn, $queries, -1.0, false, $requests_extra_info, true); + + $res = rpc_tl_query_result($query_ids); + + $responses_extra_info = []; + foreach ($query_ids as $q_id) { + $extra_info = extract_kphp_rpc_response_extra_info($q_id); + if (is_null($extra_info)) { + critical_error("got null rpc response extra info after processing an rpc response!"); + } + array_push($responses_extra_info, $extra_info); + } + + echo json_encode([ + "result" => $res["result"], + "requests_extra_info" => array_map(fn($req_tup): array => [$req_tup[0]], $requests_extra_info->get()), + "responses_extra_info" => array_map(fn($resp_tup): array => [$resp_tup[0], $resp_tup[1]], $responses_extra_info), + ]); +} + +function test_kphp_typed_rpc_extra_info() { + $queries = [new engine_stat(), new engine_pid(), new engine_version(), new engine_sleep(200)]; + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"]); + $requests_extra_info = new \KphpRpcRequestsExtraInfo; + + $query_ids = typed_rpc_tl_query($conn, $queries, -1.0, false, $requests_extra_info, true); + + $res = typed_rpc_tl_query_result($query_ids); + + $responses_extra_info = []; + foreach ($query_ids as $q_id) { + $extra_info = extract_kphp_rpc_response_extra_info($q_id); + if (is_null($extra_info)) { + critical_error("got null rpc response extra info after processing an rpc response!"); + } + array_push($responses_extra_info, $extra_info); + } + + foreach ($res as $resp) { + if ($resp->isError()) { + critical_error("bad rpc response"); + } + } + + echo json_encode([ + "requests_extra_info" => array_map(fn($req_tup): array => [$req_tup[0]], $requests_extra_info->get()), + "responses_extra_info" => array_map(fn($resp_tup): array => [$resp_tup[0], $resp_tup[1]], $responses_extra_info), + ]); +} diff --git a/tests/python/tests/rpc/php/rpc_wrappers.php b/tests/python/tests/rpc/php/rpc_wrappers.php new file mode 100644 index 0000000000..ff81854a37 --- /dev/null +++ b/tests/python/tests/rpc/php/rpc_wrappers.php @@ -0,0 +1,161 @@ + $resp->error_code]); + return; + } + + critical_error("unreachable"); +} + +function test_rpc_no_wrappers_with_ignore_answer() { + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"]); + + $query_ids = typed_rpc_tl_query($conn, [new engine_stat()], -1.0, true); + $resp = typed_rpc_tl_query_result($query_ids)[0]; + + if ($resp instanceof rpcResponseError) { + echo json_encode(["error" => $resp->error_code]); + return; + } + + critical_error("unreachable"); +} + +function test_rpc_dest_actor_with_actor_id() { + $actor_id = 1997; + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"], 0); + + $query_ids = typed_rpc_tl_query($conn, [new rpcDestActor($actor_id, new engine_stat())]); + $resp = typed_rpc_tl_query_result($query_ids)[0]; + + if ($resp instanceof rpcResponseError) { + echo json_encode(["error" => $resp->error_code]); + return; + } + + critical_error("unreachable"); +} + +function test_rpc_dest_actor_with_ignore_answer() { + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"], 0); + + $query_ids = typed_rpc_tl_query($conn, [new rpcDestActor(0, new engine_stat())], -1.0, true); + $resp = typed_rpc_tl_query_result($query_ids)[0]; + + if ($resp instanceof rpcResponseError) { + echo json_encode(["error" => $resp->error_code]); + return; + } + + critical_error("unreachable"); +} + +function test_rpc_dest_flags_with_actor_id() { + $actor_id = 1997; + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"], $actor_id); + + $query_ids = typed_rpc_tl_query($conn, [new rpcDestFlags(0, new rpcInvokeReqExtra(), new engine_stat())]); + $resp = typed_rpc_tl_query_result($query_ids)[0]; + + if ($resp instanceof rpcResponseError) { + echo json_encode(["error" => $resp->error_code]); + return; + } + + critical_error("unreachable"); +} + +function test_rpc_dest_flags_with_ignore_answer() { + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"], 0); + + $query_ids = typed_rpc_tl_query($conn, [new rpcDestFlags(0, new rpcInvokeReqExtra(), new engine_stat())], -1.0, true); + $resp = typed_rpc_tl_query_result($query_ids)[0]; + + if ($resp instanceof rpcResponseError) { + echo json_encode(["error" => $resp->error_code]); + return; + } + + critical_error("unreachable"); +} + +function test_rpc_dest_flags_with_ignore_answer_1() { + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"], 0); + + $query_ids = typed_rpc_tl_query($conn, [new rpcDestFlags(rpcInvokeReqExtra::BIT_NO_RESULT_7, new rpcInvokeReqExtra(), new engine_stat())], -1.0, true); + $resp = typed_rpc_tl_query_result($query_ids)[0]; + + if ($resp instanceof rpcResponseError) { + echo json_encode(["error" => $resp->error_code]); + return; + } + + critical_error("unreachable"); +} + +function test_rpc_dest_actor_flags_with_actor_id() { + $actor_id = 1997; + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"], $actor_id); + + $query_ids = typed_rpc_tl_query($conn, [new rpcDestActorFlags(0, 0, new rpcInvokeReqExtra(), new engine_stat())]); + $resp = typed_rpc_tl_query_result($query_ids)[0]; + + if ($resp instanceof rpcResponseError) { + echo json_encode(["error" => $resp->error_code]); + return; + } + + critical_error("unreachable"); +} + +function test_rpc_dest_actor_flags_with_ignore_answer() { + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"], 0); + + $query_ids = typed_rpc_tl_query($conn, [new rpcDestActorFlags(0, 0, new rpcInvokeReqExtra(), new engine_stat())], -1.0, true); + $resp = typed_rpc_tl_query_result($query_ids)[0]; + + if ($resp instanceof rpcResponseError) { + echo json_encode(["error" => $resp->error_code]); + return; + } + + critical_error("unreachable"); +} + +function test_rpc_dest_actor_flags_with_ignore_answer_1() { + $conn = new_rpc_connection("localhost", (int)$_GET["master-port"], 0); + + $query_ids = typed_rpc_tl_query($conn, [new rpcDestActorFlags(0, rpcInvokeReqExtra::BIT_NO_RESULT_7, new rpcInvokeReqExtra(), new engine_stat())], -1.0, true); + $resp = typed_rpc_tl_query_result($query_ids)[0]; + + if ($resp instanceof rpcResponseError) { + echo json_encode(["error" => $resp->error_code]); + return; + } + + critical_error("unreachable"); +} diff --git a/tests/python/tests/rpc/test_rpc_extra_info.py b/tests/python/tests/rpc/test_rpc_extra_info.py index d2ddb27611..c0d0ab03ec 100644 --- a/tests/python/tests/rpc/test_rpc_extra_info.py +++ b/tests/python/tests/rpc/test_rpc_extra_info.py @@ -6,7 +6,7 @@ class TestRpcExtraInfo(KphpServerAutoTestCase): def test_untyped_rpc_extra_info(self): rpc_extra_info = self.kphp_server.http_get( - "/get_kphp_untyped_rpc_extra_info?master-port={}".format(self.kphp_server.master_port)) + "/test_kphp_untyped_rpc_extra_info?master-port={}".format(self.kphp_server.master_port)) self.assertEqual(rpc_extra_info.status_code, 200) self.assertNotEqual(rpc_extra_info.text, "") @@ -33,7 +33,7 @@ def test_untyped_rpc_extra_info(self): def test_typed_rpc_extra_info(self): rpc_extra_info = self.kphp_server.http_get( - "/get_kphp_typed_rpc_extra_info?master-port={}".format(self.kphp_server.master_port)) + "/test_kphp_typed_rpc_extra_info?master-port={}".format(self.kphp_server.master_port)) self.assertEqual(rpc_extra_info.status_code, 200) self.assertNotEqual(rpc_extra_info.text, "") diff --git a/tests/python/tests/rpc/test_rpc_wrappers.py b/tests/python/tests/rpc/test_rpc_wrappers.py new file mode 100644 index 0000000000..ebd126f820 --- /dev/null +++ b/tests/python/tests/rpc/test_rpc_wrappers.py @@ -0,0 +1,109 @@ +import json + +from python.lib.testcase import KphpServerAutoTestCase + +BAD_ACTOR_ID_ERROR_CODE = -2002 +WRONG_QUERY_ID_ERROR_CODE = -1003 + + +class TestRpcWrappers(KphpServerAutoTestCase): + def test_rpc_no_wrappers_with_actor_id(self): + rpc_response = self.kphp_server.http_get("/test_rpc_no_wrappers_with_actor_id?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], BAD_ACTOR_ID_ERROR_CODE) + + def test_rpc_no_wrappers_with_ignore_answer(self): + rpc_response = self.kphp_server.http_get("/test_rpc_no_wrappers_with_ignore_answer?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], WRONG_QUERY_ID_ERROR_CODE) + + def test_rpc_dest_actor_with_actor_id(self): + rpc_response = self.kphp_server.http_get("/test_rpc_dest_actor_with_actor_id?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], BAD_ACTOR_ID_ERROR_CODE) + + def test_rpc_dest_actor_with_ignore_answer(self): + rpc_response = self.kphp_server.http_get("/test_rpc_dest_actor_with_ignore_answer?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], WRONG_QUERY_ID_ERROR_CODE) + + def test_rpc_dest_flags_with_actor_id(self): + bad_actor_id_error_code = -2002 + rpc_response = self.kphp_server.http_get("/test_rpc_dest_flags_with_actor_id?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], BAD_ACTOR_ID_ERROR_CODE) + + def test_rpc_dest_flags_with_ignore_answer(self): + rpc_response = self.kphp_server.http_get("/test_rpc_dest_flags_with_ignore_answer?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], WRONG_QUERY_ID_ERROR_CODE) + + def test_rpc_dest_flags_with_ignore_answer_1(self): + rpc_response = self.kphp_server.http_get("/test_rpc_dest_flags_with_ignore_answer_1?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], WRONG_QUERY_ID_ERROR_CODE) + + def test_rpc_dest_actor_flags_with_actor_id(self): + rpc_response = self.kphp_server.http_get("/test_rpc_dest_actor_flags_with_actor_id?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], BAD_ACTOR_ID_ERROR_CODE) + + def test_rpc_dest_actor_flags_with_ignore_answer(self): + rpc_response = self.kphp_server.http_get("/test_rpc_dest_actor_flags_with_ignore_answer?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], WRONG_QUERY_ID_ERROR_CODE) + + def test_rpc_dest_actor_flags_with_ignore_answer_1(self): + rpc_response = self.kphp_server.http_get("/test_rpc_dest_actor_flags_with_ignore_answer_1?master-port={}".format(self.kphp_server.master_port)) + + self.assertEqual(rpc_response.status_code, 200) + self.assertNotEqual(rpc_response.text, "") + + output = json.loads(rpc_response.text) + + self.assertEqual(output["error"], WRONG_QUERY_ID_ERROR_CODE) diff --git a/vkext/vkext-rpc.cpp b/vkext/vkext-rpc.cpp index 71a5a1d3d0..6f5512327c 100644 --- a/vkext/vkext-rpc.cpp +++ b/vkext/vkext-rpc.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "common/rpc-headers.h" #include "common/crc32.h" @@ -1353,11 +1354,22 @@ static int rpc_write(struct rpc_connection *c, long long qid, double timeout, bo return -1; } - RpcExtraHeaders extra_headers{}; - size_t extra_headers_size = fill_extra_headers_if_needed(extra_headers, *reinterpret_cast(outbuf->rptr), c->default_actor_id, ignore_answer); + const auto [opt_new_wrapper, cur_wrapper_size, opt_actor_id_warning_info, opt_ignore_result_warning_msg]{ + regularize_wrappers(outbuf->rptr, c->default_actor_id, ignore_answer)}; - outbuf->rptr -= extra_headers_size; - memcpy(outbuf->rptr, &extra_headers, extra_headers_size); + if (opt_actor_id_warning_info.has_value()) { + const auto [msg, cur_wrapper_actor_id, new_wrapper_actor_id]{opt_actor_id_warning_info.value()}; + php_error_docref(nullptr, E_WARNING, msg, cur_wrapper_actor_id, new_wrapper_actor_id); + } + if (opt_ignore_result_warning_msg != nullptr) { + php_error_docref(nullptr, E_WARNING, opt_ignore_result_warning_msg); + } + + if (opt_new_wrapper.has_value()) { + const auto [new_wrapper, new_wrapper_size]{opt_new_wrapper.value()}; + outbuf->rptr -= new_wrapper_size - cur_wrapper_size; + std::memcpy(outbuf->rptr, &new_wrapper, new_wrapper_size); + } unsigned crc32 = 0; int len = sizeof(RpcHeaders) + sizeof(crc32) + (outbuf->wptr - outbuf->rptr); From 114dc0b7b5b0ba9be111b2fd8a6330906a2631a1 Mon Sep 17 00:00:00 2001 From: Mikhail Kornaukhov Date: Mon, 6 May 2024 11:31:48 +0300 Subject: [PATCH 16/89] Copy sources from private ml_experiments repo --- runtime/kphp_ml/kml-files-reader.cpp | 353 +++++++++++++++++++++++++ runtime/kphp_ml/kml-files-reader.h | 13 + runtime/kphp_ml/kphp_ml.cpp | 52 ++++ runtime/kphp_ml/kphp_ml.h | 69 +++++ runtime/kphp_ml/kphp_ml_catboost.cpp | 378 +++++++++++++++++++++++++++ runtime/kphp_ml/kphp_ml_catboost.h | 146 +++++++++++ runtime/kphp_ml/kphp_ml_xgboost.cpp | 274 +++++++++++++++++++ runtime/kphp_ml/kphp_ml_xgboost.h | 98 +++++++ runtime/runtime.cmake | 7 + 9 files changed, 1390 insertions(+) create mode 100644 runtime/kphp_ml/kml-files-reader.cpp create mode 100644 runtime/kphp_ml/kml-files-reader.h create mode 100644 runtime/kphp_ml/kphp_ml.cpp create mode 100644 runtime/kphp_ml/kphp_ml.h create mode 100644 runtime/kphp_ml/kphp_ml_catboost.cpp create mode 100644 runtime/kphp_ml/kphp_ml_catboost.h create mode 100644 runtime/kphp_ml/kphp_ml_xgboost.cpp create mode 100644 runtime/kphp_ml/kphp_ml_xgboost.h diff --git a/runtime/kphp_ml/kml-files-reader.cpp b/runtime/kphp_ml/kml-files-reader.cpp new file mode 100644 index 0000000000..733e3562ce --- /dev/null +++ b/runtime/kphp_ml/kml-files-reader.cpp @@ -0,0 +1,353 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +// ATTENTION! +// This file exists both in KPHP and in a private vkcom repo "ml_experiments". +// They are almost identical, besides include paths and input types (`array` vs `unordered_map`). + +#include "kml-files-reader.h" + +#include + +/* + * .kml files unite xgboost and catboost (prediction only, not learning). + * This module reads .kml files into kphp_ml::MLModel. + * + * In KPHP, .kml files are read at master process start up + * and used in read-only mode by all worker processes when a backend calls ML inference. + * + * There is also kml-files-writer.cpp, very similar to a reader. + * But it's not a part of KPHP, it's a part of "ml_experiments" repo. + */ + +namespace { + +// This class is used to free memory in the end of the program +// It is utilized to allocate memory for 'offset_in_vec' and 'reindex_map_int2int' +template +class GlobalLifetimeResource { + std::vector managed_resources; + + ~GlobalLifetimeResource() { + for (T *ptr : managed_resources) { + delete[] ptr; + } + } + +public: + static T *acquire(size_t size) { + static GlobalLifetimeResource data; + + data.managed_resources.push_back(new T[size]); + return data.managed_resources.back(); + } +}; + +class KmlFileReader { + std::ifstream fi; + +public: + explicit KmlFileReader(const std::string &kml_filename) { + fi = std::ifstream(kml_filename, std::ios::binary); + if (!fi.is_open()) { + throw std::invalid_argument("can't open " + kml_filename + " for reading"); + } + } + + ~KmlFileReader() { + fi.close(); + } + + int read_int() noexcept { + int v; + fi.read((char *)&v, sizeof(int)); + return v; + } + + void read_int(int &v) noexcept { + fi.read((char *)&v, sizeof(int)); + } + + void read_uint32(uint32_t &v) noexcept { + fi.read((char *)&v, sizeof(uint32_t)); + } + + void read_uint64(uint64_t &v) noexcept { + fi.read((char *)&v, sizeof(uint64_t)); + } + + void read_float(float &v) noexcept { + fi.read((char *)&v, sizeof(float)); + } + + void read_double(double &v) noexcept { + fi.read((char *)&v, sizeof(double)); + } + + template + void read_enum(T &v) noexcept { + static_assert(sizeof(T) == sizeof(int)); + fi.read((char *)&v, sizeof(int)); + } + + void read_string(std::string &v) noexcept { + int len; + fi.read((char *)&len, sizeof(int)); + v.resize(len); + fi.read(v.data(), len); + } + + void read_bool(bool &v) noexcept { + v = static_cast(read_int()); + } + + void read_bytes(void *v, size_t len) noexcept { + fi.read((char *)v, static_cast(len)); + } + + template + void read_vec(std::vector &v) { + static_assert(std::is_standard_layout_v && std::is_trivial_v); + int sz; + read_int(sz); + v.resize(sz); + read_bytes(v.data(), sz * sizeof(T)); + } + + template + void read_2d_vec(std::vector> &v) { + int sz; + read_int(sz); + v.resize(sz); + for (std::vector &elem: v) { + read_vec(elem); + } + } + + void check_not_eof() const { + if (fi.is_open() && fi.eof()) { + throw std::invalid_argument("unexpected eof"); + } + } +}; + +template<> +void KmlFileReader::read_vec(std::vector &v) { + int sz = read_int(); + v.resize(sz); + for (auto &str : v) { + read_string(str); + } +} + +void kml_read_catboost_field(KmlFileReader &f, kphp_ml_catboost::CatboostProjection &v) { + f.read_vec(v.transposed_cat_feature_indexes); + f.read_vec(v.binarized_indexes); +} + +void kml_read_catboost_field(KmlFileReader &f, kphp_ml_catboost::CatboostModelCtr &v) { + f.read_uint64(v.base_hash); + f.read_enum(v.base_ctr_type); + f.read_int(v.target_border_idx); + f.read_float(v.prior_num); + f.read_float(v.prior_denom); + f.read_float(v.shift); + f.read_float(v.scale); +} + +void kml_read_catboost_field(KmlFileReader &f, kphp_ml_catboost::CatboostCompressedModelCtr &v) { + kml_read_catboost_field(f, v.projection); + + int sz = f.read_int(); + v.model_ctrs.resize(sz); + for (auto &item: v.model_ctrs) { + kml_read_catboost_field(f, item); + } +} + +void kml_read_catboost_field(KmlFileReader &f, kphp_ml_catboost::CatboostCtrValueTable &v) { + int sz = f.read_int(); + v.index_hash_viewer.reserve(sz); + for (int i = 0; i < sz; ++i) { + uint64_t k; + f.read_uint64(k); + f.read_uint32(v.index_hash_viewer[k]); + } + + f.read_int(v.target_classes_count); + f.read_int(v.counter_denominator); + f.read_vec(v.ctr_mean_history); + f.read_vec(v.ctr_total); +} + +void kml_write_catboost_field(KmlFileReader &f, kphp_ml_catboost::CatboostCtrData &v) { + int sz = f.read_int(); + v.learn_ctrs.reserve(sz); + for (int i = 0; i < sz; ++i) { + uint64_t key; + f.read_uint64(key); + kml_read_catboost_field(f, v.learn_ctrs[key]); + } +} + +void kml_read_catboost_field(KmlFileReader &f, kphp_ml_catboost::CatboostModelCtrsContainer &v) { + bool has; + f.read_bool(has); + + if (has) { + f.read_int(v.used_model_ctrs_count); + + int sz = f.read_int(); + v.compressed_model_ctrs.resize(sz); + for (auto &item: v.compressed_model_ctrs) { + kml_read_catboost_field(f, item); + } + + kml_write_catboost_field(f, v.ctr_data); + } +} + +// ------------- + +void kml_file_read_xgboost_trees_no_cat(KmlFileReader &f, [[maybe_unused]] int version, kphp_ml_xgboost::XgboostModel &xgb) { + f.read_enum(xgb.tparam_objective); + f.read_bytes(&xgb.calibration, sizeof(kphp_ml_xgboost::CalibrationMethod)); + f.read_float(xgb.base_score); + f.read_int(xgb.num_features_trained); + f.read_int(xgb.num_features_present); + f.read_int(xgb.max_required_features); + + f.check_not_eof(); + if (xgb.num_features_present <= 0 || xgb.num_features_present > xgb.max_required_features) { + throw std::invalid_argument("wrong num_features_present"); + } + + int num_trees = f.read_int(); + if (num_trees <= 0 || num_trees > 10000) { + throw std::invalid_argument("wrong num_trees"); + } + + xgb.trees.resize(num_trees); + for (kphp_ml_xgboost::XgbTree &tree: xgb.trees) { + int num_nodes = f.read_int(); + if (num_nodes <= 0 || num_nodes > 10000) { + throw std::invalid_argument("wrong num_nodes"); + } + tree.nodes.resize(num_nodes); + f.read_bytes(tree.nodes.data(), sizeof(kphp_ml_xgboost::XgbTreeNode) * num_nodes); + } + + f.check_not_eof(); + xgb.offset_in_vec = GlobalLifetimeResource::acquire(xgb.max_required_features); + f.read_bytes(xgb.offset_in_vec, xgb.max_required_features * sizeof(int)); + f.check_not_eof(); + xgb.reindex_map_int2int = GlobalLifetimeResource::acquire(xgb.max_required_features); + f.read_bytes(xgb.reindex_map_int2int, xgb.max_required_features * sizeof(int)); + f.check_not_eof(); + + int num_reindex_str2int = f.read_int(); + if (num_reindex_str2int < 0 || num_reindex_str2int > xgb.max_required_features) { + throw std::invalid_argument("wrong num_reindex_str2int"); + } + for (int i = 0; i < num_reindex_str2int; ++i) { + uint64_t hash; + f.read_uint64(hash); + f.read_int(xgb.reindex_map_str2int[hash]); + } + f.check_not_eof(); + + f.read_bool(xgb.skip_zeroes); + f.read_float(xgb.default_missing_value); +} + +void kml_file_read_catboost_trees(KmlFileReader &f, [[maybe_unused]] int version, kphp_ml_catboost::CatboostModel &cbm) { + f.read_int(cbm.float_feature_count); + f.read_int(cbm.cat_feature_count); + f.read_int(cbm.binary_feature_count); + f.read_int(cbm.tree_count); + f.check_not_eof(); + + f.read_vec(cbm.float_features_index); + f.read_2d_vec(cbm.float_feature_borders); + f.read_vec(cbm.tree_depth); + f.read_vec(cbm.one_hot_cat_feature_index); + f.read_2d_vec(cbm.one_hot_hash_values); + f.read_2d_vec(cbm.ctr_feature_borders); + f.check_not_eof(); + + f.read_vec(cbm.tree_split); + f.read_vec(cbm.leaf_values); + f.read_2d_vec(cbm.leaf_values_vec); + f.check_not_eof(); + + f.read_double(cbm.scale); + f.read_double(cbm.bias); + f.read_vec(cbm.biases); + f.read_int(cbm.dimension); + f.check_not_eof(); + + int cat_hashes_sz = f.read_int(); + cbm.cat_features_hashes.reserve(cat_hashes_sz); + for (int i = 0; i < cat_hashes_sz; ++i) { + uint64_t key_hash; + f.read_uint64(key_hash); + f.read_int(cbm.cat_features_hashes[key_hash]); + } + + f.check_not_eof(); + kml_read_catboost_field(f, cbm.model_ctrs); + + int reindex_sz = f.read_int(); + cbm.reindex_map_floats_and_cat.reserve(reindex_sz); + for (int i = 0; i < reindex_sz; ++i) { + uint64_t hash; + f.read_uint64(hash); + f.read_int(cbm.reindex_map_floats_and_cat[hash]); + } +} + +} // namespace + +kphp_ml::MLModel kml_file_read(const std::string &kml_filename) { + kphp_ml::MLModel kml; + KmlFileReader f(kml_filename); + + if (f.read_int() != kphp_ml::KML_FILE_PREFIX) { + throw std::invalid_argument("wrong .kml file prefix"); + } + int version = f.read_int(); + if (version != kphp_ml::KML_FILE_VERSION_100) { + throw std::invalid_argument("wrong .kml file version"); + } + + f.read_enum(kml.model_kind); + f.read_enum(kml.input_kind); + f.read_string(kml.model_name); + f.read_vec(kml.feature_names); + + int custom_props_sz = f.read_int(); + kml.custom_properties.reserve(custom_props_sz); + for (int i = 0; i < custom_props_sz; ++i) { + std::string property_name; + f.read_string(property_name); + f.read_string(kml.custom_properties[property_name]); + } + + f.check_not_eof(); + + switch (kml.model_kind) { + case kphp_ml::ModelKind::xgboost_trees_no_cat: + kml.impl = kphp_ml_xgboost::XgboostModel(); + kml_file_read_xgboost_trees_no_cat(f, version, std::get(kml.impl)); + break; + case kphp_ml::ModelKind::catboost_trees: + kml.impl = kphp_ml_catboost::CatboostModel(); + kml_file_read_catboost_trees(f, version, std::get(kml.impl)); + break; + default: + throw std::invalid_argument("unsupported model_kind"); + } + + return kml; +} diff --git a/runtime/kphp_ml/kml-files-reader.h b/runtime/kphp_ml/kml-files-reader.h new file mode 100644 index 0000000000..e09c535f56 --- /dev/null +++ b/runtime/kphp_ml/kml-files-reader.h @@ -0,0 +1,13 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +// ATTENTION! +// This file exists both in KPHP and in a private vkcom repo "ml_experiments". +// They are almost identical, besides include paths and input types (`array` vs `unordered_map`). + +#pragma once + +#include "kphp_ml/kphp_ml.h" + +kphp_ml::MLModel kml_file_read(const std::string &kml_filename); diff --git a/runtime/kphp_ml/kphp_ml.cpp b/runtime/kphp_ml/kphp_ml.cpp new file mode 100644 index 0000000000..af0964ca38 --- /dev/null +++ b/runtime/kphp_ml/kphp_ml.cpp @@ -0,0 +1,52 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +// ATTENTION! +// This file exists both in KPHP and in a private vkcom repo "ml_experiments". +// They are almost identical, besides include paths and input types (`array` vs `unordered_map`). + +#include "kphp_ml/kphp_ml.h" + +// for detailed comments about KML, see kphp_ml.h + +bool kphp_ml::MLModel::is_catboost_multi_classification() const { + if (!is_catboost()) { + return false; + } + + const auto &cbm = std::get(impl); + return cbm.dimension != -1 && !cbm.leaf_values_vec.empty(); +} + +unsigned int kphp_ml::MLModel::calculate_mutable_buffer_size() const { + switch (model_kind) { + case ModelKind::xgboost_trees_no_cat: { + const auto &xgb = std::get(impl); + return kphp_ml_xgboost::BATCH_SIZE_XGB * xgb.num_features_present * 2 * sizeof(float); + } + case ModelKind::catboost_trees: { + const auto &cbm = std::get(impl); + return cbm.cat_feature_count * sizeof(std::string) + + cbm.float_feature_count * sizeof(float) + + (cbm.binary_feature_count + 4 - 1) / 4 * 4 + // round up to 4 bytes + cbm.cat_feature_count * sizeof(int) + + cbm.model_ctrs.used_model_ctrs_count * sizeof(float); + } + default: + return 0; + } +} + +const std::vector &kphp_ml::MLModel::get_feature_names() const { + return feature_names; +} + +std::optional kphp_ml::MLModel::get_custom_property(const std::string &property_name) const { + auto it = custom_properties.find(property_name); + if (it == custom_properties.end()) { + return std::nullopt; + } + return it->second; +} + diff --git a/runtime/kphp_ml/kphp_ml.h b/runtime/kphp_ml/kphp_ml.h new file mode 100644 index 0000000000..4bc7a63f60 --- /dev/null +++ b/runtime/kphp_ml/kphp_ml.h @@ -0,0 +1,69 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +// ATTENTION! +// This file exists both in KPHP and in a private vkcom repo "ml_experiments". +// They are almost identical, besides include paths and input types (`array` vs `unordered_map`). + +/* +todo detailed documentation +*/ + +#pragma once + +#include +#include +#include + +#include "kphp_ml/kphp_ml_catboost.h" +#include "kphp_ml/kphp_ml_xgboost.h" + +namespace kphp_ml { + +constexpr int KML_FILE_PREFIX = 0x718249F0; +constexpr int KML_FILE_VERSION_100 = 100; + +enum class ModelKind { + invalid_kind, + xgboost_trees_no_cat, + catboost_trees, +}; + +enum class InputKind { + // [ 'user_city_99' => 1, 'user_topic_weights_17' => 7.42, ...], uses reindex_map + ht_remap_str_keys_to_fvalue, + // [ 12934 => 1, 8923 => 7.42, ... ], uses reindex_map + ht_remap_int_keys_to_fvalue, + // [ 70 => 1, 23 => 7.42, ... ], no keys reindex, pass directly + ht_direct_int_keys_to_fvalue, + + // [ 1.23, 4.56, ... ] and [ "red", "small" ]: floats and cat separately, pass directly + vectors_fvalue_and_catstr, + // the same, but a model is a catboost multiclassificator (returns an array of predictions, not one) + vectors_fvalue_and_catstr_multi, + // [ 'emb_7' => 19.98, ..., 'user_os' => 2, ... ]: in one ht, but categorials are numbers also (to avoid `mixed`) + ht_remap_str_keys_to_fvalue_or_catnum, + // the same, but multiclassificator (returns an array of predictions, not one) + ht_remap_str_keys_to_fvalue_or_catnum_multi, +}; + +struct MLModel { + ModelKind model_kind; + InputKind input_kind; + std::string model_name; + std::vector feature_names; + std::unordered_map custom_properties; + + std::variant impl; + + bool is_xgboost() const { return model_kind == ModelKind::xgboost_trees_no_cat; } + bool is_catboost() const { return model_kind == ModelKind::catboost_trees; } + bool is_catboost_multi_classification() const; + const std::vector &get_feature_names() const; + std::optional get_custom_property(const std::string &property_name) const; + + unsigned int calculate_mutable_buffer_size() const; +}; + +} // namespace kphp_ml diff --git a/runtime/kphp_ml/kphp_ml_catboost.cpp b/runtime/kphp_ml/kphp_ml_catboost.cpp new file mode 100644 index 0000000000..b22fbc88fb --- /dev/null +++ b/runtime/kphp_ml/kphp_ml_catboost.cpp @@ -0,0 +1,378 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +// ATTENTION! +// This file exists both in KPHP and in a private vkcom repo "ml_experiments". +// They are almost identical, besides include paths and input types (`array` vs `unordered_map`). + +#include "kphp_ml/kphp_ml_catboost.h" +#include "kphp_ml/kphp_ml.h" + +#include "utils/string_hash.h" +#include "testing_infra/infra_cpp.h" + +#include + + +/* + * For detailed comments about KML, see kphp_ml.h. + * + * This module contains a custom catboost predictor implementation. + * + * Note, that catboost models support not only float features, but categorial features also (ctrs). + * There are many structs related to ctrs, which are almost copied as-is from catboost sources, + * without any changes or renamings for better mapping onto original sources. + * https://github.com/catboost/catboost/blob/master/catboost/libs/model/model_export/resources/ + * + * Text features and embedded features are not supported by kml yet, we had no need of. + */ + +namespace kphp_ml_catboost { + +static inline uint64_t calc_hash(uint64_t a, uint64_t b) { + static constexpr uint64_t MAGIC_MULT = 0x4906ba494954cb65ull; + return MAGIC_MULT * (a + MAGIC_MULT * b); +} + +static uint64_t calc_hashes(const unsigned char *binarized_features, + const int *hashed_cat_features, + const std::vector &transposed_cat_feature_indexes, + const std::vector &binarized_feature_indexes) { + uint64_t result = 0; + for (int cat_feature_index: transposed_cat_feature_indexes) { + result = calc_hash(result, static_cast(hashed_cat_features[cat_feature_index])); + } + for (const CatboostBinFeatureIndexValue &bin_feature_index: binarized_feature_indexes) { + unsigned char binary_feature = binarized_features[bin_feature_index.bin_index]; + if (!bin_feature_index.check_value_equal) { + result = calc_hash(result, binary_feature >= bin_feature_index.value); + } else { + result = calc_hash(result, binary_feature == bin_feature_index.value); + } + } + return result; +} + +static void calc_ctrs(const CatboostModelCtrsContainer &model_ctrs, + const unsigned char *binarized_features, + const int *hashed_cat_features, + float *result) { + int result_index = 0; + + for (int i = 0; i < model_ctrs.compressed_model_ctrs.size(); ++i) { + const CatboostProjection &proj = model_ctrs.compressed_model_ctrs[i].projection; + uint64_t ctr_hash = calc_hashes(binarized_features, hashed_cat_features, proj.transposed_cat_feature_indexes, proj.binarized_indexes); + + for (int j = 0; j < model_ctrs.compressed_model_ctrs[i].model_ctrs.size(); ++j, ++result_index) { + const CatboostModelCtr &ctr = model_ctrs.compressed_model_ctrs[i].model_ctrs[j]; + const CatboostCtrValueTable &learn_ctr = model_ctrs.ctr_data.learn_ctrs.at(ctr.base_hash); + CatboostModelCtrType ctr_type = ctr.base_ctr_type; + const auto *bucket_ptr = learn_ctr.resolve_hash_index(ctr_hash); + if (bucket_ptr == nullptr) { + result[result_index] = ctr.calc(0, 0); + } else { + unsigned int bucket = *bucket_ptr; + if (ctr_type == CatboostModelCtrType::BinarizedTargetMeanValue || ctr_type == CatboostModelCtrType::FloatTargetMeanValue) { + const CatboostCtrMeanHistory &ctr_mean_history = learn_ctr.ctr_mean_history[bucket]; + result[result_index] = ctr.calc(ctr_mean_history.sum, static_cast(ctr_mean_history.count)); + } else if (ctr_type == CatboostModelCtrType::Counter || ctr_type == CatboostModelCtrType::FeatureFreq) { + const std::vector &ctr_total = learn_ctr.ctr_total; + result[result_index] = ctr.calc(ctr_total[bucket], learn_ctr.counter_denominator); + } else if (ctr_type == CatboostModelCtrType::Buckets) { + const std::vector &ctr_history = learn_ctr.ctr_total; + int target_classes_count = learn_ctr.target_classes_count; + int good_count = ctr_history[bucket * target_classes_count + ctr.target_border_idx]; + int total_count = 0; + for (int classId = 0; classId < target_classes_count; ++classId) { + total_count += ctr_history[bucket * target_classes_count + classId]; + } + result[result_index] = ctr.calc(good_count, total_count); + } else { + const std::vector &ctr_history = learn_ctr.ctr_total; + int target_classes_count = learn_ctr.target_classes_count; + if (target_classes_count > 2) { + int good_count = 0; + int total_count = 0; + for (int class_id = 0; class_id < ctr.target_border_idx + 1; ++class_id) { + total_count += ctr_history[bucket * target_classes_count + class_id]; + } + for (int class_id = ctr.target_border_idx + 1; class_id < target_classes_count; ++class_id) { + good_count += ctr_history[bucket * target_classes_count + class_id]; + } + total_count += good_count; + result[result_index] = ctr.calc(good_count, total_count); + } else { + result[result_index] = ctr.calc(ctr_history[bucket * 2 + 1], ctr_history[bucket * 2] + ctr_history[bucket * 2 + 1]); + } + } + } + } + } +} + +static int get_hash(const std::string &cat_feature, const std::unordered_map &cat_feature_hashes) { + auto found_it = cat_feature_hashes.find(string_hash(cat_feature.c_str(), cat_feature.size())); + return found_it == cat_feature_hashes.end() ? 0x7fffffff : found_it->second; +} + +template +static double predict_one(const CatboostModel &cbm, + const FloatOrDouble *float_features, + const std::string *cat_features, + char *mutable_buffer) { + char *p_buffer = mutable_buffer; + + // Binarise features + auto *binary_features = reinterpret_cast(mutable_buffer); + p_buffer += (cbm.binary_feature_count + 4 - 1) / 4 * 4; // round up to 4 bytes + + int binary_feature_index = -1; + + // binarize float features + for (int i = 0; i < cbm.float_feature_borders.size(); ++i) { + int cur = 0; + for (double border: cbm.float_feature_borders[i]) { + cur += float_features[cbm.float_features_index[i]] > border; + } + binary_features[++binary_feature_index] = cur; + } + + auto *transposed_hash = reinterpret_cast(p_buffer); + p_buffer += cbm.cat_feature_count * sizeof(int); + for (int i = 0; i < cbm.cat_feature_count; ++i) { + transposed_hash[i] = get_hash(cat_features[i], cbm.cat_features_hashes); + } + + // binarize one hot cat features + // note, that it slightly differs from original: one_hot_cat_feature_index is precomputed in converting to kml, no repack needed + for (int i = 0; i < cbm.one_hot_cat_feature_index.size(); ++i) { + int cat_idx = cbm.one_hot_cat_feature_index[i]; + int hash = transposed_hash[cat_idx]; + int cur = 0; + for (int border_idx = 0; border_idx < cbm.one_hot_hash_values[i].size(); ++border_idx) { + cur |= (hash == cbm.one_hot_hash_values[i][border_idx]) * (border_idx + 1); + } + binary_features[++binary_feature_index] = cur; + } + + // binarize ctr features + if (cbm.model_ctrs.used_model_ctrs_count > 0) { + auto *ctrs = reinterpret_cast(p_buffer); + p_buffer += cbm.model_ctrs.used_model_ctrs_count * sizeof(float); + calc_ctrs(cbm.model_ctrs, binary_features, transposed_hash, ctrs); + + for (int i = 0; i < cbm.ctr_feature_borders.size(); ++i) { + unsigned char cur = 0; + for (float border: cbm.ctr_feature_borders[i]) { + cur += ctrs[i] > border; + } + binary_features[++binary_feature_index] = cur; + } + } + + // Extract and sum values from trees + + double result = 0.; + int tree_splits_index = 0; + int current_tree_leaf_values_index = 0; + + for (int tree_id = 0; tree_id < cbm.tree_count; ++tree_id) { + int current_tree_depth = cbm.tree_depth[tree_id]; + int index = 0; + for (int depth = 0; depth < current_tree_depth; ++depth) { + const CatboostModel::Split &split = cbm.tree_split[tree_splits_index + depth]; + index |= ((binary_features[split.feature_index] ^ split.xor_mask) >= split.border) << depth; + } + result += cbm.leaf_values[current_tree_leaf_values_index + index]; + tree_splits_index += current_tree_depth; + current_tree_leaf_values_index += 1 << current_tree_depth; + } + + return cbm.scale * result + cbm.bias; +} + +template +static std::vector predict_one_multi(const CatboostModel &cbm, + const FloatOrDouble *float_features, + const std::string *cat_features, + char *mutable_buffer) { + char *p_buffer = mutable_buffer; + + // Binarise features + auto *binary_features = reinterpret_cast(mutable_buffer); + p_buffer += (cbm.binary_feature_count + 4 - 1) / 4 * 4; // round up to 4 bytes + + int binary_feature_index = -1; + + // binarize float features + for (int i = 0; i < cbm.float_feature_borders.size(); ++i) { + int cur = 0; + for (double border: cbm.float_feature_borders[i]) { + cur += float_features[cbm.float_features_index[i]] > border; + } + binary_features[++binary_feature_index] = cur; + } + + auto *transposed_hash = reinterpret_cast(p_buffer); + p_buffer += cbm.cat_feature_count * sizeof(int); + for (int i = 0; i < cbm.cat_feature_count; ++i) { + transposed_hash[i] = get_hash(cat_features[i], cbm.cat_features_hashes); + } + + // binarize one hot cat features + // note, that it slightly differs from original: one_hot_cat_feature_index is precomputed in converting to kml, no repack needed + for (int i = 0; i < cbm.one_hot_cat_feature_index.size(); ++i) { + int cat_idx = cbm.one_hot_cat_feature_index[i]; + int hash = transposed_hash[cat_idx]; + int cur = 0; + for (int border_idx = 0; border_idx < cbm.one_hot_hash_values[i].size(); ++border_idx) { + cur |= (hash == cbm.one_hot_hash_values[i][border_idx]) * (border_idx + 1); + } + binary_features[++binary_feature_index] = cur; + } + + // binarize ctr features + if (cbm.model_ctrs.used_model_ctrs_count > 0) { + auto *ctrs = reinterpret_cast(p_buffer); + p_buffer += cbm.model_ctrs.used_model_ctrs_count * sizeof(float); + calc_ctrs(cbm.model_ctrs, binary_features, transposed_hash, ctrs); + + for (int i = 0; i < cbm.ctr_feature_borders.size(); ++i) { + int cur = 0; + for (float border: cbm.ctr_feature_borders[i]) { + cur += ctrs[i] > border; + } + binary_features[++binary_feature_index] = cur; + } + } + + // Extract and sum values from trees + + std::vector results(cbm.dimension, 0.0); + const std::vector *leaf_values_ptr = cbm.leaf_values_vec.data(); + int tree_ptr = 0; + + for (int tree_id = 0; tree_id < cbm.tree_count; ++tree_id) { + int current_tree_depth = cbm.tree_depth[tree_id]; + int index = 0; + for (int depth = 0; depth < current_tree_depth; ++depth) { + const CatboostModel::Split &split = cbm.tree_split[tree_ptr + depth]; + index |= ((binary_features[split.feature_index] ^ split.xor_mask) >= split.border) << depth; + } + for (int i = 0; i < cbm.dimension; ++i) { + results[i] += leaf_values_ptr[index][i]; + } + tree_ptr += current_tree_depth; + leaf_values_ptr += 1 << current_tree_depth; + } + for (int i = 0; i < cbm.dimension; ++i) { + results[i] = results[i] * cbm.scale + cbm.biases[i]; + } + + return results; +} + +double kml_predict_catboost_by_vectors(const kphp_ml::MLModel &kml, + const std::vector &float_features, + const std::vector &cat_features, + char *mutable_buffer) { + const auto &cbm = std::get(kml.impl); + + if (float_features.size() < cbm.float_feature_count) { + php_warning("incorrect input size of float_features, model %s", kml.model_name.c_str()); + return 0.0; + } + if (cat_features.size() < cbm.cat_feature_count) { + php_warning("incorrect input size of cat_features, model %s", kml.model_name.c_str()); + return 0.0; + } + + return predict_one(cbm, float_features.data(), cat_features.data(), mutable_buffer); +} + +double kml_predict_catboost_by_ht_remap_str_keys(const kphp_ml::MLModel &kml, + const std::unordered_map &features_map, + char *mutable_buffer) { + const auto &cbm = std::get(kml.impl); + + auto *float_features = reinterpret_cast(mutable_buffer); + mutable_buffer += sizeof(float) * cbm.float_feature_count; + std::fill_n(float_features, cbm.float_feature_count, 0.0); + + auto *cat_features = reinterpret_cast(mutable_buffer); + mutable_buffer += sizeof(std::string) * cbm.cat_feature_count; + for (int i = 0; i < cbm.cat_feature_count; ++i) { // std::fill_n for string is unsafe + new (cat_features + i) std::string(); + } + + for (const auto &kv: features_map) { + const std::string &feature_name = kv.first; + const uint64_t key_hash = string_hash(feature_name.c_str(), feature_name.size()); + double f_or_cat = kv.second; + + if (auto found_it = cbm.reindex_map_floats_and_cat.find(key_hash); found_it != cbm.reindex_map_floats_and_cat.end()) { + int feature_id = found_it->second; + if (feature_id >= CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT) { + cat_features[feature_id - CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT] = std::to_string(static_cast(std::round(f_or_cat))); + } else { + float_features[feature_id] = static_cast(f_or_cat); + } + } + } + + return predict_one(cbm, float_features, cat_features, mutable_buffer); +} + +std::vector kml_predict_catboost_by_vectors_multi(const kphp_ml::MLModel &kml, + const std::vector &float_features, + const std::vector &cat_features, + char *mutable_buffer) { + const auto &cbm = std::get(kml.impl); + + if (float_features.size() < cbm.float_feature_count) { + php_warning("incorrect input size of float_features, model %s", kml.model_name.c_str()); + return {}; + } + if (cat_features.size() < cbm.cat_feature_count) { + php_warning("incorrect input size of cat_features, model %s", kml.model_name.c_str()); + return {}; + } + + return predict_one_multi(cbm, float_features.data(), cat_features.data(), mutable_buffer); +} + +std::vector kml_predict_catboost_by_ht_remap_str_keys_multi(const kphp_ml::MLModel &kml, + const std::unordered_map &features_map, + char *mutable_buffer) { + const auto &cbm = std::get(kml.impl); + + auto *float_features = reinterpret_cast(mutable_buffer); + mutable_buffer += sizeof(float) * cbm.float_feature_count; + std::fill_n(float_features, cbm.float_feature_count, 0.0); + + auto *cat_features = reinterpret_cast(mutable_buffer); + mutable_buffer += sizeof(std::string) * cbm.cat_feature_count; + for (int i = 0; i < cbm.cat_feature_count; ++i) { // std::fill_n for string is unsafe + new (cat_features + i) std::string(); + } + + for (const auto &kv: features_map) { + const std::string &feature_name = kv.first; + const uint64_t key_hash = string_hash(feature_name.c_str(), feature_name.size()); + double f_or_cat = kv.second; + + if (auto found_it = cbm.reindex_map_floats_and_cat.find(key_hash); found_it != cbm.reindex_map_floats_and_cat.end()) { + int feature_id = found_it->second; + if (feature_id >= CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT) { + cat_features[feature_id - CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT] = std::to_string(static_cast(std::round(f_or_cat))); + } else { + float_features[feature_id] = static_cast(f_or_cat); + } + } + } + + return predict_one_multi(cbm, float_features, cat_features, mutable_buffer); +} + +} // namespace kphp_ml_catboost diff --git a/runtime/kphp_ml/kphp_ml_catboost.h b/runtime/kphp_ml/kphp_ml_catboost.h new file mode 100644 index 0000000000..21191e85a6 --- /dev/null +++ b/runtime/kphp_ml/kphp_ml_catboost.h @@ -0,0 +1,146 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +// ATTENTION! +// This file exists both in KPHP and in a private vkcom repo "ml_experiments". +// They are almost identical, besides include paths and input types (`array` vs `unordered_map`). + +#pragma once + +#include +#include +#include +#include + + +namespace kphp_ml { struct MLModel; } +struct InputRow; + +namespace kphp_ml_catboost { + +enum class CatboostModelCtrType { + Borders, + Buckets, + BinarizedTargetMeanValue, + FloatTargetMeanValue, + Counter, + FeatureFreq, + CtrTypesCount, +}; + +struct CatboostModelCtr { + uint64_t base_hash; + CatboostModelCtrType base_ctr_type; + int target_border_idx; + float prior_num; + float prior_denom; + float shift; + float scale; + + float calc(float count_in_class, float total_count) const noexcept { + float ctr = (count_in_class + prior_num) / (total_count + prior_denom); + return (ctr + shift) * scale; + } + + float calc(int count_in_class, int total_count) const noexcept { + return calc(static_cast(count_in_class), static_cast(total_count)); + } +}; + +struct CatboostBinFeatureIndexValue { + int bin_index; + bool check_value_equal; + unsigned char value; +}; + +struct CatboostCtrMeanHistory { + float sum; + int count; +}; + +struct CatboostCtrValueTable { + std::unordered_map index_hash_viewer; + int target_classes_count; + int counter_denominator; + std::vector ctr_mean_history; + std::vector ctr_total; + + const unsigned int *resolve_hash_index(uint64_t hash) const noexcept { + auto found_it = index_hash_viewer.find(hash); + return found_it == index_hash_viewer.end() ? nullptr : &found_it->second; + } +}; + +struct CatboostCtrData { + std::unordered_map learn_ctrs; +}; + +struct CatboostProjection { + std::vector transposed_cat_feature_indexes; + std::vector binarized_indexes; +}; + +struct CatboostCompressedModelCtr { + CatboostProjection projection; + std::vector model_ctrs; +}; + +struct CatboostModelCtrsContainer { + int used_model_ctrs_count{0}; + std::vector compressed_model_ctrs; + CatboostCtrData ctr_data; +}; + +struct CatboostModel { + struct Split { + uint16_t feature_index; + uint8_t xor_mask; + uint8_t border; + }; + static_assert(sizeof(Split) == 4); + + int float_feature_count; + int cat_feature_count; + int binary_feature_count; + int tree_count; + std::vector float_features_index; + std::vector> float_feature_borders; + std::vector tree_depth; + std::vector one_hot_cat_feature_index; + std::vector> one_hot_hash_values; + std::vector> ctr_feature_borders; + + std::vector tree_split; + std::vector leaf_values; // this and below are like a union + std::vector> leaf_values_vec; + + double scale; + + double bias; // this and below are like a union + std::vector biases; + + int dimension = -1; // absent in case of NON-multiclass classification + std::unordered_map cat_features_hashes; + + CatboostModelCtrsContainer model_ctrs; + // todo there are also embedded and text features, we may want to add them later + + // to accept input_kind = ht_remap_str_keys_to_fvalue_or_catstr and similar + // 1) we store [hash => vec_idx] instead of [feature_name => vec_idx], we don't expect collisions + // 2) this reindex_map contains both reindexes of float and categorial features, but categorial are large: + // [ 'emb_7' => 7, ..., 'user_age_group' => 1000001, 'user_os' => 1000002 ] + // the purpose of storing two maps in one is to use a single hashtable lookup when remapping + // todo this ht is filled once (on .kml loading) and used many-many times for lookup; maybe, find smth faster than std + std::unordered_map reindex_map_floats_and_cat; + static constexpr int REINDEX_MAP_CATEGORIAL_SHIFT = 1000000; +}; + +double kml_predict_catboost_by_vectors(const kphp_ml::MLModel &kml, const std::vector &float_features, const std::vector &cat_features, char *mutable_buffer); +double kml_predict_catboost_by_ht_remap_str_keys(const kphp_ml::MLModel &kml, const std::unordered_map &features_map, char *mutable_buffer); + +std::vector kml_predict_catboost_by_vectors_multi(const kphp_ml::MLModel &kml, const std::vector &float_features, const std::vector &cat_features, char *mutable_buffer); +std::vector kml_predict_catboost_by_ht_remap_str_keys_multi(const kphp_ml::MLModel &kml, const std::unordered_map &features_map, char *mutable_buffer); + + +} // namespace kphp_ml_catboost diff --git a/runtime/kphp_ml/kphp_ml_xgboost.cpp b/runtime/kphp_ml/kphp_ml_xgboost.cpp new file mode 100644 index 0000000000..cc32681c28 --- /dev/null +++ b/runtime/kphp_ml/kphp_ml_xgboost.cpp @@ -0,0 +1,274 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +// ATTENTION! +// This file exists both in KPHP and in a private vkcom repo "ml_experiments". +// They are almost identical, besides include paths and input types (`array` vs `unordered_map`). + +#include "kphp_ml/kphp_ml.h" +#include "kphp_ml/kphp_ml_xgboost.h" + +#include + +#include "utils/string_hash.h" +#include "testing_infra/infra_cpp.h" + +/* + * For detailed comments about KML, see kphp_ml.h. + * + * This module contains a custom xgboost predictor implementation. + * It's much faster than native xgboost due to compressed layout and avoiding ifs in code. + */ + +namespace kphp_ml_xgboost { + +static_assert(sizeof(XgbTreeNode) == 8, "unexpected sizeof(XgbTreeNode)"); + +struct XgbDensePredictor { + [[gnu::always_inline]] static inline uint64_t default_missing_value(const XgboostModel &model) { + std::pair default_missing_value_layout; + if (std::isnan(model.default_missing_value)) { + // Should be +/- infinity, but it would be much slower, just +/- "big" value + default_missing_value_layout = {+1e10, -1e10}; + } else { + default_missing_value_layout = {model.default_missing_value, model.default_missing_value}; + } + + return *reinterpret_cast(&default_missing_value_layout); + } + + float *vector_x{nullptr}; // assigned outside as a chunk in linear memory, 2 equal values per existing feature + + typedef void (XgbDensePredictor::*filler_int_keys)(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept; + typedef void (XgbDensePredictor::*filler_str_keys)(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept; + + void fill_vector_x_ht_direct(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { + for (const auto &kv: features_map) { + const int feature_id = kv.first; + const double fvalue = kv.second; + + if (feature_id < 0 || feature_id >= xgb.max_required_features) { // unexisting index [ 100000 => 0.123 ], it's ok + continue; + } + + int vec_offset = xgb.offset_in_vec[feature_id]; + if (vec_offset != -1) { + vector_x[vec_offset] = static_cast(fvalue); + vector_x[vec_offset + 1] = static_cast(fvalue); + } + } + } + + void fill_vector_x_ht_direct_sz(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { + for (const auto &kv: features_map) { + const int feature_id = kv.first; + const double fvalue = kv.second; + + if (feature_id < 0 || feature_id >= xgb.max_required_features) { // unexisting index [ 100000 => 0.123 ], it's ok + continue; + } + if (std::fabs(fvalue) < 1e-9) { + continue; + } + + int vec_offset = xgb.offset_in_vec[feature_id]; + if (vec_offset != -1) { + vector_x[vec_offset] = static_cast(fvalue); + vector_x[vec_offset + 1] = static_cast(fvalue); + } + } + } + + void fill_vector_x_ht_remap_str_key(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { + for (const auto &kv: features_map) { + const std::string &feature_name = kv.first; + const double fvalue = kv.second; + + auto found_it = xgb.reindex_map_str2int.find(string_hash(feature_name.c_str(), feature_name.size())); + if (found_it != xgb.reindex_map_str2int.end()) { // input contains [ "unexisting_feature" => 0.123 ], it's ok + int vec_offset = found_it->second; + vector_x[vec_offset] = static_cast(fvalue); + vector_x[vec_offset + 1] = static_cast(fvalue); + } + } + } + + void fill_vector_x_ht_remap_str_key_sz(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { + for (const auto &kv: features_map) { + const std::string &feature_name = kv.first; + const double fvalue = kv.second; + + if (std::fabs(fvalue) < 1e-9) { + continue; + } + + auto found_it = xgb.reindex_map_str2int.find(string_hash(feature_name.c_str(), feature_name.size())); + if (found_it != xgb.reindex_map_str2int.end()) { // input contains [ "unexisting_feature" => 0.123 ], it's ok + int vec_offset = found_it->second; + vector_x[vec_offset] = static_cast(fvalue); + vector_x[vec_offset + 1] = static_cast(fvalue); + } + } + } + + void fill_vector_x_ht_remap_int_key(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { + for (const auto &kv: features_map) { + const int feature_id = kv.first; + const double fvalue = kv.second; + + if (feature_id < 0 || feature_id >= xgb.max_required_features) { // unexisting index [ 100000 => 0.123 ], it's ok + continue; + } + + int vec_offset = xgb.reindex_map_int2int[feature_id]; + if (vec_offset != -1) { + vector_x[vec_offset] = static_cast(fvalue); + vector_x[vec_offset + 1] = static_cast(fvalue); + } + } + } + + void fill_vector_x_ht_remap_int_key_sz(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { + for (const auto &kv: features_map) { + const int feature_id = kv.first; + const double fvalue = kv.second; + + if (feature_id < 0 || feature_id >= xgb.max_required_features) { // unexisting index [ 100000 => 0.123 ], it's ok + continue; + } + if (std::fabs(fvalue) < 1e-9) { + continue; + } + + int vec_offset = xgb.reindex_map_int2int[feature_id]; + if (vec_offset != -1) { + vector_x[vec_offset] = static_cast(fvalue); + vector_x[vec_offset + 1] = static_cast(fvalue); + } + } + } + + float predict_one_tree(const XgbTree &tree) const noexcept { + const XgbTreeNode *node = tree.nodes.data(); + while (!node->is_leaf()) { + bool goto_right = vector_x[node->vec_offset_dense()] >= node->split_cond; + node = &tree.nodes[node->left_child() + goto_right]; + } + return node->split_cond; + } +}; + +[[gnu::always_inline]] static inline float transform_base_score(XGTrainParamObjective tparam_objective, float base_score) { + switch (tparam_objective) { + case XGTrainParamObjective::binary_logistic: + return -logf(1.0f / base_score - 1.0f); + case XGTrainParamObjective::rank_pairwise: + return base_score; + default: + __builtin_unreachable(); + } +} + +[[gnu::always_inline]] static inline double transform_prediction(XGTrainParamObjective tparam_objective, double score) { + switch (tparam_objective) { + case XGTrainParamObjective::binary_logistic: + return 1.0 / (1.0 + std::exp(-score)); + case XGTrainParamObjective::rank_pairwise: + return score; + default: + __builtin_unreachable(); + } +} + +float XgboostModel::transform_base_score() const noexcept { + return kphp_ml_xgboost::transform_base_score(tparam_objective, base_score); +} + +double XgboostModel::transform_prediction(double score) const noexcept { + score = kphp_ml_xgboost::transform_prediction(tparam_objective, score); + + switch (calibration.calibration_method) { + case CalibrationMethod::platt_scaling: + if (tparam_objective == XGTrainParamObjective::binary_logistic) { + score = -log(1. / score - 1); + } + return 1 / (1. + exp(-(calibration.platt_slope * score + calibration.platt_intercept))); + case CalibrationMethod::update_prior: + return score * calibration.new_prior * (1 - calibration.old_prior) / + (calibration.old_prior * (1 - score - calibration.new_prior) + score * calibration.new_prior); + default: + return score; + } +} + +std::vector kml_predict_xgboost(const kphp_ml::MLModel &kml, + const std::vector &in, + char *mutable_buffer) { + const auto &xgb = std::get(kml.impl); + int n_rows = static_cast(in.size()); + + XgbDensePredictor::filler_int_keys filler_int_keys{nullptr}; + XgbDensePredictor::filler_str_keys filler_str_keys{nullptr}; + switch (kml.input_kind) { + case kphp_ml::InputKind::ht_direct_int_keys_to_fvalue: + filler_int_keys = xgb.skip_zeroes ? &XgbDensePredictor::fill_vector_x_ht_direct_sz : &XgbDensePredictor::fill_vector_x_ht_direct; + break; + case kphp_ml::InputKind::ht_remap_int_keys_to_fvalue: + filler_int_keys = xgb.skip_zeroes ? &XgbDensePredictor::fill_vector_x_ht_remap_int_key_sz : &XgbDensePredictor::fill_vector_x_ht_remap_int_key; + break; + case kphp_ml::InputKind::ht_remap_str_keys_to_fvalue: + filler_str_keys = xgb.skip_zeroes ? &XgbDensePredictor::fill_vector_x_ht_remap_str_key_sz : &XgbDensePredictor::fill_vector_x_ht_remap_str_key; + break; + default: + assert(0 && "invalid input_kind"); + return {}; + } + + XgbDensePredictor feat_vecs[BATCH_SIZE_XGB]; + for (int i = 0; i < BATCH_SIZE_XGB && i < n_rows; ++i) { + feat_vecs[i].vector_x = reinterpret_cast(mutable_buffer) + i * xgb.num_features_present * 2; + } + auto iter_done = in.begin(); + + const float base_score = xgb.transform_base_score(); + std::vector out_predictions; + out_predictions.reserve(n_rows); + for (int i = 0; i < n_rows; ++i) { + out_predictions.push_back(base_score); + } + + const int n_batches = static_cast(std::ceil(static_cast(n_rows) / BATCH_SIZE_XGB)); + + for (int block_id = 0; block_id < n_batches; ++block_id) { + const int batch_offset = block_id * BATCH_SIZE_XGB; + const int block_size = std::min(n_rows - batch_offset, BATCH_SIZE_XGB); + + std::fill_n(reinterpret_cast(mutable_buffer), block_size * xgb.num_features_present, XgbDensePredictor::default_missing_value(xgb)); + if (filler_int_keys != nullptr) { + for (int i = 0; i < block_size; ++i) { + (feat_vecs[i].*filler_int_keys)(xgb, iter_done->ht_int_keys_to_fvalue); + ++iter_done; + } + } else { + for (int i = 0; i < block_size; ++i) { + (feat_vecs[i].*filler_str_keys)(xgb, iter_done->ht_str_keys_to_fvalue); + ++iter_done; + } + } + + for (const XgbTree &tree: xgb.trees) { + for (int i = 0; i < block_size; ++i) { + out_predictions[batch_offset + i] += feat_vecs[i].predict_one_tree(tree); + } + } + } + + for (int i = 0; i < n_rows; ++i) { + out_predictions[i] = xgb.transform_prediction(out_predictions[i]); + } + + return out_predictions; +} + +} // namespace kphp_ml_xgboost diff --git a/runtime/kphp_ml/kphp_ml_xgboost.h b/runtime/kphp_ml/kphp_ml_xgboost.h new file mode 100644 index 0000000000..25ac1a0ade --- /dev/null +++ b/runtime/kphp_ml/kphp_ml_xgboost.h @@ -0,0 +1,98 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +// ATTENTION! +// This file exists both in KPHP and in a private vkcom repo "ml_experiments". +// They are almost identical, besides include paths and input types (`array` vs `unordered_map`). + +#pragma once + +#include +#include + +namespace kphp_ml { struct MLModel; } +struct InputRow; + +namespace kphp_ml_xgboost { + +constexpr int BATCH_SIZE_XGB = 8; + +enum class XGTrainParamObjective { + binary_logistic, + rank_pairwise, +}; + +struct CalibrationMethod { + enum { + no_calibration, + platt_scaling, // https://en.wikipedia.org/wiki/Platt_scaling + update_prior, // https://www.mpia.de/3432751/probcomb_TN.pdf, formulae 12, 18 + } calibration_method{no_calibration}; + + union { + struct { + // for calibration_method = platt_scaling + double platt_slope; + double platt_intercept; + }; + struct { + // for calibration_method = update_prior + double old_prior; + double new_prior; + }; + }; +}; + +struct XgbTreeNode { + // first 16 bits: vec_offset = vec_idx * 2 + (1 if default left) + // last 16 bits: left child node index (right = left + 1, always) + int combined_value; + float split_cond; + + bool is_leaf() const noexcept { return combined_value < 0; } + int vec_offset_dense() const noexcept { return combined_value & 0xFFFF; } + int vec_offset_sparse() const noexcept { return (combined_value & 0xFFFF) >> 1; } + int left_child() const noexcept { return combined_value >> 16; } + bool default_left() const noexcept { return combined_value & 1; } +}; + +struct XgbTree { + std::vector nodes; +}; + +struct XgboostModel { + XGTrainParamObjective tparam_objective; + CalibrationMethod calibration; + float base_score; + int num_features_trained{0}; + int num_features_present{0}; + int max_required_features{0}; + + std::vector trees; + + // to accept input_kind = ht_remap_str_keys_to_fvalue + // note, that the main optimization is in storing + // [hash => vec_offset] instead of [string => vec_offset], we don't expect collisions + // todo this ht is filled once (on .kml loading) and used many-many times for lookup; maybe, find smth faster than std + std::unordered_map reindex_map_str2int; + // to accept input_kind = ht_remap_int_keys_to_fvalue + // see below, same format + int *reindex_map_int2int; + // to accept input_kind = ht_direct_int_keys_to_fvalue + // looks like [-1, vec_offset, -1, -1, ...] + // any feature_id can be looked up as offset_in_vec[feature_id]: + // * -1 means "feature is not used in a model (in any tree)" + // * otherwise, it's used to access vector_x, see XgbDensePredictor + int *offset_in_vec; + + bool skip_zeroes; + float default_missing_value; + + float transform_base_score() const noexcept; + double transform_prediction(double score) const noexcept; +}; + +std::vector kml_predict_xgboost(const kphp_ml::MLModel &kml, const std::vector &in, char *mutable_buffer); + +} // namespace kphp_ml_xgboost diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index fdcc227b92..c9ada104f5 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -29,6 +29,12 @@ prepend(KPHP_RUNTIME_JOB_WORKERS_SOURCES job-workers/ processing-jobs.cpp server-functions.cpp) +prepend(KPHP_RUNTIME_ML_SOURCES kphp_ml/ + kphp_ml.cpp + kphp_ml_catboost.cpp + kphp_ml_xgboost.cpp + kml-files-reader.cpp) + prepend(KPHP_RUNTIME_SPL_SOURCES spl/ array_iterator.cpp) @@ -55,6 +61,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ ${KPHP_RUNTIME_MSGPACK_SOURCES} ${KPHP_RUNTIME_JOB_WORKERS_SOURCES} ${KPHP_RUNTIME_SPL_SOURCES} + ${KPHP_RUNTIME_ML_SOURCES} ${KPHP_RUNTIME_PDO_SOURCES} ${KPHP_RUNTIME_PDO_MYSQL_SOURCES} ${KPHP_RUNTIME_PDO_PGSQL_SOURCES} From 10dd1b086eff01d440703c116ec523379eb7f109 Mon Sep 17 00:00:00 2001 From: Mikhail Kornaukhov Date: Mon, 6 May 2024 11:41:39 +0300 Subject: [PATCH 17/89] Update sources from ml_experiments, now they are compilable --- runtime/kphp_ml/kml-files-reader.cpp | 2 +- runtime/kphp_ml/kml-files-reader.h | 2 +- runtime/kphp_ml/kphp_ml.cpp | 6 +- runtime/kphp_ml/kphp_ml.h | 4 +- runtime/kphp_ml/kphp_ml_catboost.cpp | 98 +++++++++++++++------------- runtime/kphp_ml/kphp_ml_catboost.h | 11 ++-- runtime/kphp_ml/kphp_ml_xgboost.cpp | 95 ++++++++++++++++----------- runtime/kphp_ml/kphp_ml_xgboost.h | 13 +++- 8 files changed, 131 insertions(+), 100 deletions(-) diff --git a/runtime/kphp_ml/kml-files-reader.cpp b/runtime/kphp_ml/kml-files-reader.cpp index 733e3562ce..14df99a745 100644 --- a/runtime/kphp_ml/kml-files-reader.cpp +++ b/runtime/kphp_ml/kml-files-reader.cpp @@ -6,7 +6,7 @@ // This file exists both in KPHP and in a private vkcom repo "ml_experiments". // They are almost identical, besides include paths and input types (`array` vs `unordered_map`). -#include "kml-files-reader.h" +#include "runtime/kphp_ml/kml-files-reader.h" #include diff --git a/runtime/kphp_ml/kml-files-reader.h b/runtime/kphp_ml/kml-files-reader.h index e09c535f56..a742a150ad 100644 --- a/runtime/kphp_ml/kml-files-reader.h +++ b/runtime/kphp_ml/kml-files-reader.h @@ -8,6 +8,6 @@ #pragma once -#include "kphp_ml/kphp_ml.h" +#include "runtime/kphp_ml/kphp_ml.h" kphp_ml::MLModel kml_file_read(const std::string &kml_filename); diff --git a/runtime/kphp_ml/kphp_ml.cpp b/runtime/kphp_ml/kphp_ml.cpp index af0964ca38..b73c775969 100644 --- a/runtime/kphp_ml/kphp_ml.cpp +++ b/runtime/kphp_ml/kphp_ml.cpp @@ -6,7 +6,8 @@ // This file exists both in KPHP and in a private vkcom repo "ml_experiments". // They are almost identical, besides include paths and input types (`array` vs `unordered_map`). -#include "kphp_ml/kphp_ml.h" +#include "runtime/kphp_core.h" +#include "runtime/kphp_ml/kphp_ml.h" // for detailed comments about KML, see kphp_ml.h @@ -27,7 +28,7 @@ unsigned int kphp_ml::MLModel::calculate_mutable_buffer_size() const { } case ModelKind::catboost_trees: { const auto &cbm = std::get(impl); - return cbm.cat_feature_count * sizeof(std::string) + + return cbm.cat_feature_count * sizeof(string) + cbm.float_feature_count * sizeof(float) + (cbm.binary_feature_count + 4 - 1) / 4 * 4 + // round up to 4 bytes cbm.cat_feature_count * sizeof(int) + @@ -49,4 +50,3 @@ std::optional kphp_ml::MLModel::get_custom_property(const std::stri } return it->second; } - diff --git a/runtime/kphp_ml/kphp_ml.h b/runtime/kphp_ml/kphp_ml.h index 4bc7a63f60..55eab69578 100644 --- a/runtime/kphp_ml/kphp_ml.h +++ b/runtime/kphp_ml/kphp_ml.h @@ -16,8 +16,8 @@ todo detailed documentation #include #include -#include "kphp_ml/kphp_ml_catboost.h" -#include "kphp_ml/kphp_ml_xgboost.h" +#include "runtime/kphp_ml/kphp_ml_catboost.h" +#include "runtime/kphp_ml/kphp_ml_xgboost.h" namespace kphp_ml { diff --git a/runtime/kphp_ml/kphp_ml_catboost.cpp b/runtime/kphp_ml/kphp_ml_catboost.cpp index b22fbc88fb..24ccaa712e 100644 --- a/runtime/kphp_ml/kphp_ml_catboost.cpp +++ b/runtime/kphp_ml/kphp_ml_catboost.cpp @@ -6,14 +6,10 @@ // This file exists both in KPHP and in a private vkcom repo "ml_experiments". // They are almost identical, besides include paths and input types (`array` vs `unordered_map`). -#include "kphp_ml/kphp_ml_catboost.h" -#include "kphp_ml/kphp_ml.h" - -#include "utils/string_hash.h" -#include "testing_infra/infra_cpp.h" - -#include +#include "runtime/kphp_ml/kphp_ml_catboost.h" +#include "runtime/kphp_core.h" +#include "runtime/kphp_ml/kphp_ml.h" /* * For detailed comments about KML, see kphp_ml.h. @@ -111,7 +107,7 @@ static void calc_ctrs(const CatboostModelCtrsContainer &model_ctrs, } } -static int get_hash(const std::string &cat_feature, const std::unordered_map &cat_feature_hashes) { +static int get_hash(const string &cat_feature, const std::unordered_map &cat_feature_hashes) { auto found_it = cat_feature_hashes.find(string_hash(cat_feature.c_str(), cat_feature.size())); return found_it == cat_feature_hashes.end() ? 0x7fffffff : found_it->second; } @@ -119,7 +115,7 @@ static int get_hash(const std::string &cat_feature, const std::unordered_map static double predict_one(const CatboostModel &cbm, const FloatOrDouble *float_features, - const std::string *cat_features, + const string *cat_features, char *mutable_buffer) { char *p_buffer = mutable_buffer; @@ -193,10 +189,10 @@ static double predict_one(const CatboostModel &cbm, } template -static std::vector predict_one_multi(const CatboostModel &cbm, - const FloatOrDouble *float_features, - const std::string *cat_features, - char *mutable_buffer) { +static array predict_one_multi(const CatboostModel &cbm, + const FloatOrDouble *float_features, + const string *cat_features, + char *mutable_buffer) { char *p_buffer = mutable_buffer; // Binarise features @@ -249,7 +245,11 @@ static std::vector predict_one_multi(const CatboostModel &cbm, // Extract and sum values from trees - std::vector results(cbm.dimension, 0.0); + array results(array_size(cbm.dimension, true)); + for (int i = 0; i < cbm.dimension; ++i) { + results[i] = 0.0; + } + const std::vector *leaf_values_ptr = cbm.leaf_values_vec.data(); int tree_ptr = 0; @@ -274,25 +274,25 @@ static std::vector predict_one_multi(const CatboostModel &cbm, } double kml_predict_catboost_by_vectors(const kphp_ml::MLModel &kml, - const std::vector &float_features, - const std::vector &cat_features, + const array &float_features, + const array &cat_features, char *mutable_buffer) { const auto &cbm = std::get(kml.impl); - if (float_features.size() < cbm.float_feature_count) { - php_warning("incorrect input size of float_features, model %s", kml.model_name.c_str()); + if (!float_features.is_vector() || float_features.count() < cbm.float_feature_count) { + php_warning("incorrect input size for float_features, model %s", kml.model_name.c_str()); return 0.0; } - if (cat_features.size() < cbm.cat_feature_count) { - php_warning("incorrect input size of cat_features, model %s", kml.model_name.c_str()); + if (!cat_features.is_vector() || cat_features.count() < cbm.cat_feature_count) { + php_warning("incorrect input size for cat_features, model %s", kml.model_name.c_str()); return 0.0; } - return predict_one(cbm, float_features.data(), cat_features.data(), mutable_buffer); + return predict_one(cbm, float_features.get_const_vector_pointer(), cat_features.get_const_vector_pointer(), mutable_buffer); } double kml_predict_catboost_by_ht_remap_str_keys(const kphp_ml::MLModel &kml, - const std::unordered_map &features_map, + const array &features_map, char *mutable_buffer) { const auto &cbm = std::get(kml.impl); @@ -300,21 +300,24 @@ double kml_predict_catboost_by_ht_remap_str_keys(const kphp_ml::MLModel &kml, mutable_buffer += sizeof(float) * cbm.float_feature_count; std::fill_n(float_features, cbm.float_feature_count, 0.0); - auto *cat_features = reinterpret_cast(mutable_buffer); - mutable_buffer += sizeof(std::string) * cbm.cat_feature_count; - for (int i = 0; i < cbm.cat_feature_count; ++i) { // std::fill_n for string is unsafe - new (cat_features + i) std::string(); + auto *cat_features = reinterpret_cast(mutable_buffer); + mutable_buffer += sizeof(string) * cbm.cat_feature_count; + for (int i = 0; i < cbm.cat_feature_count; ++i) { + new (cat_features + i) string(); } for (const auto &kv: features_map) { - const std::string &feature_name = kv.first; + if (__builtin_expect(!kv.is_string_key(), false)) { + continue; + } + const string &feature_name = kv.get_string_key(); const uint64_t key_hash = string_hash(feature_name.c_str(), feature_name.size()); - double f_or_cat = kv.second; + double f_or_cat = kv.get_value(); if (auto found_it = cbm.reindex_map_floats_and_cat.find(key_hash); found_it != cbm.reindex_map_floats_and_cat.end()) { int feature_id = found_it->second; if (feature_id >= CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT) { - cat_features[feature_id - CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT] = std::to_string(static_cast(std::round(f_or_cat))); + cat_features[feature_id - CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT] = f$strval(static_cast(std::round(f_or_cat))); } else { float_features[feature_id] = static_cast(f_or_cat); } @@ -324,48 +327,51 @@ double kml_predict_catboost_by_ht_remap_str_keys(const kphp_ml::MLModel &kml, return predict_one(cbm, float_features, cat_features, mutable_buffer); } -std::vector kml_predict_catboost_by_vectors_multi(const kphp_ml::MLModel &kml, - const std::vector &float_features, - const std::vector &cat_features, - char *mutable_buffer) { +array kml_predict_catboost_by_vectors_multi(const kphp_ml::MLModel &kml, + const array &float_features, + const array &cat_features, + char *mutable_buffer) { const auto &cbm = std::get(kml.impl); - if (float_features.size() < cbm.float_feature_count) { + if (!float_features.is_vector() || float_features.count() < cbm.float_feature_count) { php_warning("incorrect input size of float_features, model %s", kml.model_name.c_str()); return {}; } - if (cat_features.size() < cbm.cat_feature_count) { + if (!cat_features.is_vector() || cat_features.count() < cbm.cat_feature_count) { php_warning("incorrect input size of cat_features, model %s", kml.model_name.c_str()); return {}; } - return predict_one_multi(cbm, float_features.data(), cat_features.data(), mutable_buffer); + return predict_one_multi(cbm, float_features.get_const_vector_pointer(), cat_features.get_const_vector_pointer(), mutable_buffer); } -std::vector kml_predict_catboost_by_ht_remap_str_keys_multi(const kphp_ml::MLModel &kml, - const std::unordered_map &features_map, - char *mutable_buffer) { +array kml_predict_catboost_by_ht_remap_str_keys_multi(const kphp_ml::MLModel &kml, + const array &features_map, + char *mutable_buffer) { const auto &cbm = std::get(kml.impl); auto *float_features = reinterpret_cast(mutable_buffer); mutable_buffer += sizeof(float) * cbm.float_feature_count; std::fill_n(float_features, cbm.float_feature_count, 0.0); - auto *cat_features = reinterpret_cast(mutable_buffer); - mutable_buffer += sizeof(std::string) * cbm.cat_feature_count; - for (int i = 0; i < cbm.cat_feature_count; ++i) { // std::fill_n for string is unsafe - new (cat_features + i) std::string(); + auto *cat_features = reinterpret_cast(mutable_buffer); + mutable_buffer += sizeof(string) * cbm.cat_feature_count; + for (int i = 0; i < cbm.cat_feature_count; ++i) { + new (cat_features + i) string(); } for (const auto &kv: features_map) { - const std::string &feature_name = kv.first; + if (__builtin_expect(!kv.is_string_key(), false)) { + continue; + } + const string &feature_name = kv.get_string_key(); const uint64_t key_hash = string_hash(feature_name.c_str(), feature_name.size()); - double f_or_cat = kv.second; + double f_or_cat = kv.get_value(); if (auto found_it = cbm.reindex_map_floats_and_cat.find(key_hash); found_it != cbm.reindex_map_floats_and_cat.end()) { int feature_id = found_it->second; if (feature_id >= CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT) { - cat_features[feature_id - CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT] = std::to_string(static_cast(std::round(f_or_cat))); + cat_features[feature_id - CatboostModel::REINDEX_MAP_CATEGORIAL_SHIFT] = f$strval(static_cast(std::round(f_or_cat))); } else { float_features[feature_id] = static_cast(f_or_cat); } diff --git a/runtime/kphp_ml/kphp_ml_catboost.h b/runtime/kphp_ml/kphp_ml_catboost.h index 21191e85a6..2214225587 100644 --- a/runtime/kphp_ml/kphp_ml_catboost.h +++ b/runtime/kphp_ml/kphp_ml_catboost.h @@ -10,12 +10,11 @@ #include #include -#include #include +#include "runtime/kphp_core.h" namespace kphp_ml { struct MLModel; } -struct InputRow; namespace kphp_ml_catboost { @@ -136,11 +135,11 @@ struct CatboostModel { static constexpr int REINDEX_MAP_CATEGORIAL_SHIFT = 1000000; }; -double kml_predict_catboost_by_vectors(const kphp_ml::MLModel &kml, const std::vector &float_features, const std::vector &cat_features, char *mutable_buffer); -double kml_predict_catboost_by_ht_remap_str_keys(const kphp_ml::MLModel &kml, const std::unordered_map &features_map, char *mutable_buffer); +double kml_predict_catboost_by_vectors(const kphp_ml::MLModel &kml, const array &float_features, const array &cat_features, char *mutable_buffer); +double kml_predict_catboost_by_ht_remap_str_keys(const kphp_ml::MLModel &kml, const array &features_map, char *mutable_buffer); -std::vector kml_predict_catboost_by_vectors_multi(const kphp_ml::MLModel &kml, const std::vector &float_features, const std::vector &cat_features, char *mutable_buffer); -std::vector kml_predict_catboost_by_ht_remap_str_keys_multi(const kphp_ml::MLModel &kml, const std::unordered_map &features_map, char *mutable_buffer); +array kml_predict_catboost_by_vectors_multi(const kphp_ml::MLModel &kml, const array &float_features, const array &cat_features, char *mutable_buffer); +array kml_predict_catboost_by_ht_remap_str_keys_multi(const kphp_ml::MLModel &kml, const array &features_map, char *mutable_buffer); } // namespace kphp_ml_catboost diff --git a/runtime/kphp_ml/kphp_ml_xgboost.cpp b/runtime/kphp_ml/kphp_ml_xgboost.cpp index cc32681c28..82764ba0ad 100644 --- a/runtime/kphp_ml/kphp_ml_xgboost.cpp +++ b/runtime/kphp_ml/kphp_ml_xgboost.cpp @@ -6,13 +6,12 @@ // This file exists both in KPHP and in a private vkcom repo "ml_experiments". // They are almost identical, besides include paths and input types (`array` vs `unordered_map`). -#include "kphp_ml/kphp_ml.h" -#include "kphp_ml/kphp_ml_xgboost.h" +#include "runtime/kphp_ml/kphp_ml_xgboost.h" #include -#include "utils/string_hash.h" -#include "testing_infra/infra_cpp.h" +#include "runtime/kphp_core.h" +#include "runtime/kphp_ml/kphp_ml.h" /* * For detailed comments about KML, see kphp_ml.h. @@ -40,13 +39,16 @@ struct XgbDensePredictor { float *vector_x{nullptr}; // assigned outside as a chunk in linear memory, 2 equal values per existing feature - typedef void (XgbDensePredictor::*filler_int_keys)(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept; - typedef void (XgbDensePredictor::*filler_str_keys)(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept; + typedef void (XgbDensePredictor::*filler_int_keys)(const XgboostModel &xgb, const array &features_map) const noexcept; + typedef void (XgbDensePredictor::*filler_str_keys)(const XgboostModel &xgb, const array &features_map) const noexcept; - void fill_vector_x_ht_direct(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { - for (const auto &kv: features_map) { - const int feature_id = kv.first; - const double fvalue = kv.second; + void fill_vector_x_ht_direct(const XgboostModel &xgb, const array &features_map) const noexcept { + for (const auto &kv : features_map) { + if (__builtin_expect(kv.is_string_key(), false)) { + continue; + } + const int feature_id = kv.get_int_key(); + const double fvalue = kv.get_value(); if (feature_id < 0 || feature_id >= xgb.max_required_features) { // unexisting index [ 100000 => 0.123 ], it's ok continue; @@ -60,10 +62,13 @@ struct XgbDensePredictor { } } - void fill_vector_x_ht_direct_sz(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { - for (const auto &kv: features_map) { - const int feature_id = kv.first; - const double fvalue = kv.second; + void fill_vector_x_ht_direct_sz(const XgboostModel &xgb, const array &features_map) const noexcept { + for (const auto &kv : features_map) { + if (__builtin_expect(kv.is_string_key(), false)) { + continue; + } + const int feature_id = kv.get_int_key(); + const double fvalue = kv.get_value(); if (feature_id < 0 || feature_id >= xgb.max_required_features) { // unexisting index [ 100000 => 0.123 ], it's ok continue; @@ -80,10 +85,13 @@ struct XgbDensePredictor { } } - void fill_vector_x_ht_remap_str_key(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { - for (const auto &kv: features_map) { - const std::string &feature_name = kv.first; - const double fvalue = kv.second; + void fill_vector_x_ht_remap_str_key(const XgboostModel &xgb, const array &features_map) const noexcept { + for (const auto &kv : features_map) { + if (__builtin_expect(!kv.is_string_key(), false)) { + continue; + } + const string &feature_name = kv.get_string_key(); + const double fvalue = kv.get_value(); auto found_it = xgb.reindex_map_str2int.find(string_hash(feature_name.c_str(), feature_name.size())); if (found_it != xgb.reindex_map_str2int.end()) { // input contains [ "unexisting_feature" => 0.123 ], it's ok @@ -94,10 +102,13 @@ struct XgbDensePredictor { } } - void fill_vector_x_ht_remap_str_key_sz(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { - for (const auto &kv: features_map) { - const std::string &feature_name = kv.first; - const double fvalue = kv.second; + void fill_vector_x_ht_remap_str_key_sz(const XgboostModel &xgb, const array &features_map) const noexcept { + for (const auto &kv : features_map) { + if (__builtin_expect(!kv.is_string_key(), false)) { + continue; + } + const string &feature_name = kv.get_string_key(); + const double fvalue = kv.get_value(); if (std::fabs(fvalue) < 1e-9) { continue; @@ -112,10 +123,13 @@ struct XgbDensePredictor { } } - void fill_vector_x_ht_remap_int_key(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { - for (const auto &kv: features_map) { - const int feature_id = kv.first; - const double fvalue = kv.second; + void fill_vector_x_ht_remap_int_key(const XgboostModel &xgb, const array &features_map) const noexcept { + for (const auto &kv : features_map) { + if (__builtin_expect(kv.is_string_key(), false)) { + continue; + } + const int feature_id = kv.get_int_key(); + const double fvalue = kv.get_value(); if (feature_id < 0 || feature_id >= xgb.max_required_features) { // unexisting index [ 100000 => 0.123 ], it's ok continue; @@ -129,10 +143,13 @@ struct XgbDensePredictor { } } - void fill_vector_x_ht_remap_int_key_sz(const XgboostModel &xgb, const std::unordered_map &features_map) const noexcept { - for (const auto &kv: features_map) { - const int feature_id = kv.first; - const double fvalue = kv.second; + void fill_vector_x_ht_remap_int_key_sz(const XgboostModel &xgb, const array &features_map) const noexcept { + for (const auto &kv : features_map) { + if (__builtin_expect(kv.is_string_key(), false)) { + continue; + } + const int feature_id = kv.get_int_key(); + const double fvalue = kv.get_value(); if (feature_id < 0 || feature_id >= xgb.max_required_features) { // unexisting index [ 100000 => 0.123 ], it's ok continue; @@ -202,11 +219,11 @@ double XgboostModel::transform_prediction(double score) const noexcept { } } -std::vector kml_predict_xgboost(const kphp_ml::MLModel &kml, - const std::vector &in, - char *mutable_buffer) { +array kml_predict_xgboost(const kphp_ml::MLModel &kml, + const array> &in, + char *mutable_buffer) { const auto &xgb = std::get(kml.impl); - int n_rows = static_cast(in.size()); + int n_rows = static_cast(in.size().size); XgbDensePredictor::filler_int_keys filler_int_keys{nullptr}; XgbDensePredictor::filler_str_keys filler_str_keys{nullptr}; @@ -232,8 +249,8 @@ std::vector kml_predict_xgboost(const kphp_ml::MLModel &kml, auto iter_done = in.begin(); const float base_score = xgb.transform_base_score(); - std::vector out_predictions; - out_predictions.reserve(n_rows); + array out_predictions; + out_predictions.reserve(n_rows, true); for (int i = 0; i < n_rows; ++i) { out_predictions.push_back(base_score); } @@ -247,17 +264,17 @@ std::vector kml_predict_xgboost(const kphp_ml::MLModel &kml, std::fill_n(reinterpret_cast(mutable_buffer), block_size * xgb.num_features_present, XgbDensePredictor::default_missing_value(xgb)); if (filler_int_keys != nullptr) { for (int i = 0; i < block_size; ++i) { - (feat_vecs[i].*filler_int_keys)(xgb, iter_done->ht_int_keys_to_fvalue); + (feat_vecs[i].*filler_int_keys)(xgb, iter_done.get_value()); ++iter_done; } } else { for (int i = 0; i < block_size; ++i) { - (feat_vecs[i].*filler_str_keys)(xgb, iter_done->ht_str_keys_to_fvalue); + (feat_vecs[i].*filler_str_keys)(xgb, iter_done.get_value()); ++iter_done; } } - for (const XgbTree &tree: xgb.trees) { + for (const XgbTree &tree : xgb.trees) { for (int i = 0; i < block_size; ++i) { out_predictions[batch_offset + i] += feat_vecs[i].predict_one_tree(tree); } diff --git a/runtime/kphp_ml/kphp_ml_xgboost.h b/runtime/kphp_ml/kphp_ml_xgboost.h index 25ac1a0ade..bd124ce77d 100644 --- a/runtime/kphp_ml/kphp_ml_xgboost.h +++ b/runtime/kphp_ml/kphp_ml_xgboost.h @@ -11,8 +11,16 @@ #include #include +#include "runtime/kphp_core.h" + +/* + * For detailed comments about KML, see kphp_ml.h. + * + * This module contains a custom xgboost predictor implementation. + * It's much faster than native xgboost due to compressed layout and avoiding ifs in code. + */ + namespace kphp_ml { struct MLModel; } -struct InputRow; namespace kphp_ml_xgboost { @@ -86,6 +94,7 @@ struct XgboostModel { // * otherwise, it's used to access vector_x, see XgbDensePredictor int *offset_in_vec; + // for ModelKind::xgboost_ht_remap bool skip_zeroes; float default_missing_value; @@ -93,6 +102,6 @@ struct XgboostModel { double transform_prediction(double score) const noexcept; }; -std::vector kml_predict_xgboost(const kphp_ml::MLModel &kml, const std::vector &in, char *mutable_buffer); +array kml_predict_xgboost(const kphp_ml::MLModel &kml, const array> &in, char *mutable_buffer); } // namespace kphp_ml_xgboost From eb6bc253d066a706e42aa16dc319bd8ae92586d7 Mon Sep 17 00:00:00 2001 From: Mikhail Kornaukhov Date: Mon, 6 May 2024 11:57:46 +0300 Subject: [PATCH 18/89] Integrate kml into kphp runtime and create php wrappers --- builtin-functions/_functions.txt | 1 + builtin-functions/kml.txt | 14 +++ runtime/interface.cpp | 2 + runtime/kphp_ml/kphp_ml_init.cpp | 94 +++++++++++++++ runtime/kphp_ml/kphp_ml_init.h | 18 +++ runtime/kphp_ml/kphp_ml_interface.cpp | 160 ++++++++++++++++++++++++++ runtime/kphp_ml/kphp_ml_interface.h | 20 ++++ runtime/runtime.cmake | 2 + server/php-engine.cpp | 19 +++ 9 files changed, 330 insertions(+) create mode 100644 builtin-functions/kml.txt create mode 100644 runtime/kphp_ml/kphp_ml_init.cpp create mode 100644 runtime/kphp_ml/kphp_ml_init.h create mode 100644 runtime/kphp_ml/kphp_ml_interface.cpp create mode 100644 runtime/kphp_ml/kphp_ml_interface.h diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index a191a3120b..ef816fff19 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -19,6 +19,7 @@ require_once __DIR__ . '/kphp_tracing.txt'; require_once __DIR__ . '/uberh3.txt'; require_once __DIR__ . '/spl.txt'; require_once __DIR__ . '/ffi.txt'; +require_once __DIR__ . '/kml.txt'; require_once __DIR__ . '/pdo/PDO.php'; require_once __DIR__ . '/pdo/PDOStatement.php'; diff --git a/builtin-functions/kml.txt b/builtin-functions/kml.txt new file mode 100644 index 0000000000..cd5cabff1a --- /dev/null +++ b/builtin-functions/kml.txt @@ -0,0 +1,14 @@ +::get().reset_json_logs_count(); worker_global_init_handlers(worker_type); vk::singleton::get().init(); + init_kphp_ml_runtime_in_worker(); } void read_engine_tag(const char *file_name) { diff --git a/runtime/kphp_ml/kphp_ml_init.cpp b/runtime/kphp_ml/kphp_ml_init.cpp new file mode 100644 index 0000000000..23117cc238 --- /dev/null +++ b/runtime/kphp_ml/kphp_ml_init.cpp @@ -0,0 +1,94 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime/kphp_ml/kphp_ml_init.h" + +#include +#include +#include +#include +#include + +#include "common/kprintf.h" +#include "runtime/kphp_ml/kml-files-reader.h" + +const char *kml_directory = nullptr; +std::unordered_map loaded_models; +unsigned int max_mutable_buffer_size = 0; + +char *mutable_buffer_in_worker = nullptr; + +static bool ends_with(const char *str, const char *suffix) { + size_t len_str = strlen(str); + size_t len_suffix = strlen(suffix); + + return len_suffix <= len_str && strncmp(str + len_str - len_suffix, suffix, len_suffix) == 0; +} + +static void load_kml_file(const std::string &path) { + kphp_ml::MLModel kml; + try { + kml = kml_file_read(path); + } catch (const std::exception &ex) { + kprintf("error reading %s: %s\n", path.c_str(), ex.what()); + return; + } + + unsigned int buffer_size = kml.calculate_mutable_buffer_size(); + max_mutable_buffer_size = std::max(max_mutable_buffer_size, (buffer_size + 3) & -4); + + uint64_t key_hash = string_hash(kml.model_name.c_str(), kml.model_name.size()); + if (auto dup_it = loaded_models.find(key_hash); dup_it != loaded_models.end()) { + kprintf("warning: model_name '%s' is duplicated\n", kml.model_name.c_str()); + } + + loaded_models[key_hash] = std::move(kml); +} + +static void traverse_kml_dir(const std::string &path) { + if (ends_with(path.c_str(), ".kml")) { + load_kml_file(path); + return; + } + + static auto is_directory = [](const char *s) { + struct stat st; + return stat(s, &st) == 0 && S_ISDIR(st.st_mode); + }; + + if (is_directory(path.c_str())) { + DIR *dir = opendir(path.c_str()); + struct dirent *iter; + while ((iter = readdir(dir))) { + if (strcmp(iter->d_name, ".") == 0 || strcmp(iter->d_name, "..") == 0) continue; + traverse_kml_dir(path + "/" + iter->d_name); + } + closedir(dir); + } +} + +void init_kphp_ml_runtime_in_master() { + if (kml_directory == nullptr || kml_directory[0] == '\0') { + return; + } + + traverse_kml_dir(kml_directory); + + kprintf("loaded %d kml models from %s\n", static_cast(loaded_models.size()), kml_directory); +} + +void init_kphp_ml_runtime_in_worker() { + mutable_buffer_in_worker = new char[max_mutable_buffer_size]; +} + +char *kphp_ml_get_mutable_buffer_in_current_worker() { + return mutable_buffer_in_worker; +} + +const kphp_ml::MLModel *kphp_ml_find_loaded_model_by_name(const string &model_name) { + uint64_t key_hash = string_hash(model_name.c_str(), model_name.size()); + auto found_it = loaded_models.find(key_hash); + + return found_it == loaded_models.end() ? nullptr : &found_it->second; +} diff --git a/runtime/kphp_ml/kphp_ml_init.h b/runtime/kphp_ml/kphp_ml_init.h new file mode 100644 index 0000000000..4843f785c8 --- /dev/null +++ b/runtime/kphp_ml/kphp_ml_init.h @@ -0,0 +1,18 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime/kphp_core.h" + +namespace kphp_ml { struct MLModel; } + +extern const char *kml_directory; + +void init_kphp_ml_runtime_in_master(); +void init_kphp_ml_runtime_in_worker(); + + +char *kphp_ml_get_mutable_buffer_in_current_worker(); +const kphp_ml::MLModel *kphp_ml_find_loaded_model_by_name(const string &model_name); diff --git a/runtime/kphp_ml/kphp_ml_interface.cpp b/runtime/kphp_ml/kphp_ml_interface.cpp new file mode 100644 index 0000000000..315c89e54a --- /dev/null +++ b/runtime/kphp_ml/kphp_ml_interface.cpp @@ -0,0 +1,160 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime/kphp_ml/kphp_ml_interface.h" + +#include "runtime/critical_section.h" +#include "runtime/kphp_ml/kphp_ml.h" +#include "runtime/kphp_ml/kphp_ml_catboost.h" +#include "runtime/kphp_ml/kphp_ml_init.h" +#include "runtime/kphp_ml/kphp_ml_xgboost.h" + +Optional> f$kml_xgboost_predict_matrix(const string &model_name, const array> &features_map_matrix) { + const kphp_ml::MLModel *p_kml = kphp_ml_find_loaded_model_by_name(model_name); + if (p_kml == nullptr) { + php_warning("kml model %s not found", model_name.c_str()); + return {}; + } + if (!p_kml->is_xgboost()) { + php_warning("called for non-xgboost model %s", model_name.c_str()); + return {}; + } + if (!features_map_matrix.is_vector()) { + php_warning("expecting a vector of hashmaps, but hashmap given"); + return {}; + } + + char *mutable_buffer = kphp_ml_get_mutable_buffer_in_current_worker(); + return kphp_ml_xgboost::kml_predict_xgboost(*p_kml, features_map_matrix, mutable_buffer); +} + +Optional f$kml_catboost_predict_vectors(const string &model_name, const array &float_features, const array &cat_features) { + const kphp_ml::MLModel *p_kml = kphp_ml_find_loaded_model_by_name(model_name); + if (p_kml == nullptr) { + php_warning("kml model %s not found", model_name.c_str()); + return {}; + } + if (!p_kml->is_catboost()) { + php_warning("called for non-catboost model %s", model_name.c_str()); + return {}; + } + if (p_kml->is_catboost_multi_classification()) { + php_warning("called for MULTI-classification model %s", model_name.c_str()); + return {}; + } + if (!float_features.is_vector() || !cat_features.is_vector()) { + php_warning("expecting two vectors, but hashmap given"); + return {}; + } + + char *mutable_buffer = kphp_ml_get_mutable_buffer_in_current_worker(); + return kphp_ml_catboost::kml_predict_catboost_by_vectors(*p_kml, float_features, cat_features, mutable_buffer); +} + +Optional f$kml_catboost_predict_ht(const string &model_name, const array &features_map) { + const kphp_ml::MLModel *p_kml = kphp_ml_find_loaded_model_by_name(model_name); + if (p_kml == nullptr) { + php_warning("kml model %s not found", model_name.c_str()); + return {}; + } + if (!p_kml->is_catboost()) { + php_warning("called for non-catboost model %s", model_name.c_str()); + return {}; + } + if (p_kml->is_catboost_multi_classification()) { + php_warning("called for MULTI-classification model %s", model_name.c_str()); + return {}; + } + if (features_map.is_vector()) { + php_warning("expecting a hashmap, but vector given"); + return {}; + } + + char *mutable_buffer = kphp_ml_get_mutable_buffer_in_current_worker(); + return kphp_ml_catboost::kml_predict_catboost_by_ht_remap_str_keys(*p_kml, features_map, mutable_buffer); +} + +Optional> f$kml_catboost_predict_vectors_multi(const string &model_name, const array &float_features, const array &cat_features) { + const kphp_ml::MLModel *p_kml = kphp_ml_find_loaded_model_by_name(model_name); + if (p_kml == nullptr) { + php_warning("kml model %s not found", model_name.c_str()); + return {}; + } + if (!p_kml->is_catboost()) { + php_warning("called for non-catboost model %s", model_name.c_str()); + return {}; + } + if (!p_kml->is_catboost_multi_classification()) { + php_warning("called for NOT MULTI-classification model %s", model_name.c_str()); + return {}; + } + if (!float_features.is_vector() || !cat_features.is_vector()) { + php_warning("expecting two vectors, but hashmap given"); + return {}; + } + + char *mutable_buffer = kphp_ml_get_mutable_buffer_in_current_worker(); + return kphp_ml_catboost::kml_predict_catboost_by_vectors_multi(*p_kml, float_features, cat_features, mutable_buffer); +} + +Optional> f$kml_catboost_predict_ht_multi(const string &model_name, const array &features_map) { + const kphp_ml::MLModel *p_kml = kphp_ml_find_loaded_model_by_name(model_name); + if (p_kml == nullptr) { + php_warning("kml model %s not found", model_name.c_str()); + return {}; + } + if (!p_kml->is_catboost()) { + php_warning("called for non-catboost model %s", model_name.c_str()); + return {}; + } + if (!p_kml->is_catboost_multi_classification()) { + php_warning("called for NOT multi-classification model %s", model_name.c_str()); + return {}; + } + if (features_map.is_vector()) { + php_warning("expecting a hashmap, but vector given"); + return {}; + } + + char *mutable_buffer = kphp_ml_get_mutable_buffer_in_current_worker(); + return kphp_ml_catboost::kml_predict_catboost_by_ht_remap_str_keys_multi(*p_kml, features_map, mutable_buffer); +} + +bool f$kml_model_exists(const string &model_name) { + return kphp_ml_find_loaded_model_by_name(model_name) != nullptr; +} + +Optional> f$kml_get_feature_names(const string &model_name) { + const kphp_ml::MLModel *p_kml = kphp_ml_find_loaded_model_by_name(model_name); + if (p_kml == nullptr) { + php_warning("kml model %s not found", model_name.c_str()); + return {}; + } + const auto &feature_names = p_kml->get_feature_names(); + + array response; + response.reserve(feature_names.size(), true); + + for (const std::string &feature_name : feature_names) { + response.emplace_back(feature_name.c_str()); + } + + return response; +} + +Optional f$kml_get_custom_property(const string &model_name, const string &property_name) { + const kphp_ml::MLModel *p_kml = kphp_ml_find_loaded_model_by_name(model_name); + if (p_kml == nullptr) { + php_warning("kml model %s not found", model_name.c_str()); + return {}; + } + + auto guard = dl::CriticalSectionGuard(); + + auto inner_result = p_kml->get_custom_property(std::string(property_name.c_str())); + if (!inner_result.has_value()) { + return {}; + } + return string(inner_result.value().c_str()); +} diff --git a/runtime/kphp_ml/kphp_ml_interface.h b/runtime/kphp_ml/kphp_ml_interface.h new file mode 100644 index 0000000000..bb785d2173 --- /dev/null +++ b/runtime/kphp_ml/kphp_ml_interface.h @@ -0,0 +1,20 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime/kphp_core.h" + +Optional> f$kml_xgboost_predict_matrix(const string &model_name, const array> &features_map_matrix); + +Optional f$kml_catboost_predict_vectors(const string &model_name, const array &float_features, const array &cat_features); +Optional f$kml_catboost_predict_ht(const string &model_name, const array &features_map); + +Optional> f$kml_catboost_predict_vectors_multi(const string &model_name, const array &float_features, const array &cat_features); +Optional> f$kml_catboost_predict_ht_multi(const string &model_name, const array &features_map); + +bool f$kml_model_exists(const string &model_name); + +Optional> f$kml_get_feature_names(const string &model_name); +Optional f$kml_get_custom_property(const string &model_name, const string &property_name); diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index c9ada104f5..edd072a119 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -33,6 +33,8 @@ prepend(KPHP_RUNTIME_ML_SOURCES kphp_ml/ kphp_ml.cpp kphp_ml_catboost.cpp kphp_ml_xgboost.cpp + kphp_ml_init.cpp + kphp_ml_interface.cpp kml-files-reader.cpp) prepend(KPHP_RUNTIME_SPL_SOURCES spl/ diff --git a/server/php-engine.cpp b/server/php-engine.cpp index 3befc48bb4..06f8273e90 100644 --- a/server/php-engine.cpp +++ b/server/php-engine.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "common/algorithms/find.h" @@ -59,6 +60,7 @@ #include "runtime/interface.h" #include "runtime/json-functions.h" +#include "runtime/kphp_ml/kphp_ml_init.h" #include "runtime/profiler.h" #include "runtime/rpc.h" #include "runtime/thread-pool.h" @@ -1679,6 +1681,9 @@ void init_all() { worker_id = (int)lrand48(); + // TODO: In the future, we want to parallelize it + init_kphp_ml_runtime_in_master(); + init_confdata_binlog_reader(); auto end_time = std::chrono::steady_clock::now(); @@ -2229,6 +2234,19 @@ int main_args_handler(int i, const char *long_option) { } return res; } + case 2040: { + static auto is_directory = [](const char* s) { + struct stat st; + return stat(s, &st) == 0 && S_ISDIR(st.st_mode); + }; + + if (!*optarg || !is_directory(optarg)) { + kprintf("--%s option: is not a directory\n", long_option); + return -1; + } + kml_directory = optarg; + return 0; + } default: return -1; } @@ -2343,6 +2361,7 @@ void parse_main_args(int argc, char *argv[]) { "Initial binlog is readed with x10 times larger timeout"); parse_option("confdata-soft-oom-ratio", required_argument, 2039, "Memory limit ratio to start ignoring new keys related events (default: 0.85)." "Can't be > hard oom ratio (0.95)"); + parse_option("kml-dir", required_argument, 2040, "Directory that contains .kml files"); parse_engine_options_long(argc, argv, main_args_handler); parse_main_args_till_option(argc, argv); From 9f136a0539b7372cb6aae494e341c2a5d6d1e96e Mon Sep 17 00:00:00 2001 From: Mikhail Kornaukhov Date: Thu, 18 Apr 2024 17:20:16 +0300 Subject: [PATCH 19/89] Add tests --- tests/python/lib/kphp_run_once.py | 4 +- tests/python/tests/kml/__init__.py | 0 ...catboost_multiclass_tutorial_ht_catnum.kml | Bin 0 -> 970 bytes .../kml/catboost_tiny_1float_1hot_10trees.kml | Bin 0 -> 624 bytes ...catboost_multiclass_tutorial_ht_catnum.php | 77 ++++++++++++++++++ .../php/catboost_tiny_1float_1hot_10trees.php | 71 ++++++++++++++++ .../catboost/php/unrelated_to_inference.php | 20 +++++ tests/python/tests/kml/test_kml.py | 36 ++++++++ .../xgb_tiny_ht_direct_int_keys_to_fvalue.kml | Bin 0 -> 197 bytes .../xgb_tiny_ht_remap_str_keys_to_fvalue.kml | Bin 0 -> 220 bytes .../xgb_tiny_ht_direct_int_keys_to_fvalue.php | 36 ++++++++ .../xgb_tiny_ht_remap_str_keys_to_fvalue.php | 36 ++++++++ 12 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 tests/python/tests/kml/__init__.py create mode 100644 tests/python/tests/kml/catboost/kml/catboost_multiclass_tutorial_ht_catnum.kml create mode 100644 tests/python/tests/kml/catboost/kml/catboost_tiny_1float_1hot_10trees.kml create mode 100644 tests/python/tests/kml/catboost/php/catboost_multiclass_tutorial_ht_catnum.php create mode 100644 tests/python/tests/kml/catboost/php/catboost_tiny_1float_1hot_10trees.php create mode 100644 tests/python/tests/kml/catboost/php/unrelated_to_inference.php create mode 100644 tests/python/tests/kml/test_kml.py create mode 100644 tests/python/tests/kml/xgboost/kml/xgb_tiny_ht_direct_int_keys_to_fvalue.kml create mode 100644 tests/python/tests/kml/xgboost/kml/xgb_tiny_ht_remap_str_keys_to_fvalue.kml create mode 100644 tests/python/tests/kml/xgboost/php/xgb_tiny_ht_direct_int_keys_to_fvalue.php create mode 100644 tests/python/tests/kml/xgboost/php/xgb_tiny_ht_remap_str_keys_to_fvalue.php diff --git a/tests/python/lib/kphp_run_once.py b/tests/python/lib/kphp_run_once.py index 485b4925bd..e40275b082 100644 --- a/tests/python/lib/kphp_run_once.py +++ b/tests/python/lib/kphp_run_once.py @@ -93,14 +93,14 @@ def run_with_php(self, extra_options=[], runs_cnt=1): return php_proc.returncode == 0 - def run_with_kphp(self, runs_cnt=1): + def run_with_kphp(self, runs_cnt=1, args=[]): self._clear_working_dir(self._kphp_runtime_tmp_dir) sanitizer_log_name = "kphp_runtime_sanitizer_log" env, sanitizer_glob_mask = self._prepare_sanitizer_env(self._kphp_runtime_tmp_dir, sanitizer_log_name) cmd = [self._kphp_runtime_bin, "--once={}".format(runs_cnt), "--profiler-log-prefix", "profiler.log", - "--worker-queries-to-reload", "1"] + "--worker-queries-to-reload", "1"] + args if not os.getuid(): cmd += ["-u", "root", "-g", "root"] kphp_server_proc = subprocess.Popen(cmd, diff --git a/tests/python/tests/kml/__init__.py b/tests/python/tests/kml/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/python/tests/kml/catboost/kml/catboost_multiclass_tutorial_ht_catnum.kml b/tests/python/tests/kml/catboost/kml/catboost_multiclass_tutorial_ht_catnum.kml new file mode 100644 index 0000000000000000000000000000000000000000..83a43c5d1666d3fac7a3733f843d4cfcb8362e10 GIT binary patch literal 970 zcmZ`%Ur1A76#q6MY!sF$S?R+@C~1soB@DZ_xH%{(EtCjE=dz)L>E?EgB!ZRn7py>` z6b0=eq8`jpkmB5z423zIIIQd;D5Le1=F1Y&zTY<;Bi3QR`#ZmLe&>95mv8CRjkZgS zF+Il?j(mZ}s<)8Pp^(9=)R zA&c-gM8~KhdPD}JpB&IX{We((-K8B}OoKit6=}}kDqTC*Bj*408|_16v>RnG{9CVo zJzCqjPgC>P#QBxp2UO|JkiqD`T};zrk!K`VT}ZjFHDZ>(7uHa7_lodGbODO!m<+~z zazKX%-c$GG`&42$B=*Ubd*y|z3O%f!6aJ1tCv^;dlfdxzj;71!shDox8yC9o%qVsF zRPvtKo$`IMbE+siXGQ{Je(~0dRCvWiGZjySzVot|rsis>bh|#~d@x)_!SEXiEZ*02 zlWfCfwq>KJFTMRD%BhLY&2$7JktV=Ve(lJ8jR4H}cIp j4o_QkUH!m_YCW!GFvGS#d&WNR{DDi{;dK1M9QOYOgUQ!} literal 0 HcmV?d00001 diff --git a/tests/python/tests/kml/catboost/kml/catboost_tiny_1float_1hot_10trees.kml b/tests/python/tests/kml/catboost/kml/catboost_tiny_1float_1hot_10trees.kml new file mode 100644 index 0000000000000000000000000000000000000000..151189e662285fbf95a78acccd446e90e7557804 GIT binary patch literal 624 zcmez1*;JUqz`(!+#LPgf2*k;WB}w`D#U=42nR%7*hG{wZi6!xd8TmlMprj}@wHTzB z5r`{5L?wvF1;hnK`30#(C6)0;JV0JqVotn~jsl1>Mv*mSg~}R&bb~O=01z7lU}6jm z2kaRb6dXX>L28gO$gLngG7VxQn*n0;PSz{~QVbGcxqttG6i6G)3NV|I0WOZhhl(@e zkjF5?0%)+Y@n$=qyZ16MY}g0nGKj1@Zewg*3={)_S^LWO&9?!u&YlUeV>r;ZZ^D8c zyA?+v{gwD;DRs_o>Pzust9-5SaYzBUt}a z7b&o~=Y!d{2UcF*8$8JbZZ1S6jQjwM8xXj#A4L8K0#M*3GB7ll$ckA0TEl!H3Cf1W H0Z1JHb2)wR literal 0 HcmV?d00001 diff --git a/tests/python/tests/kml/catboost/php/catboost_multiclass_tutorial_ht_catnum.php b/tests/python/tests/kml/catboost/php/catboost_multiclass_tutorial_ht_catnum.php new file mode 100644 index 0000000000..7543f26e42 --- /dev/null +++ b/tests/python/tests/kml/catboost/php/catboost_multiclass_tutorial_ht_catnum.php @@ -0,0 +1,77 @@ + [ + "year" => 1996, + "count" => 197, + "season" => 1 + ], + "ans" => [ + -0.4315705250918395, + -0.07602514583990287, + 0.5075956709317426 + ] + ]), + shape([ + "features_map" => [ + "year" => 1968, + "count" => 37, + "season" => 1 + ], + "ans" => [ + -0.7547556359399779, + -0.9511000859169567, + 1.7058557218569348 + ] + ]), + shape([ + "features_map" => [ + "year" => 2002, + "count" => 77, + "season" => 0 + ], + "ans" => [ + -0.1531870128758389, + 0.3682398885336728, + -0.21505287565783404 + ] + ]), + shape([ + "features_map" => [ + "year" => 1948, + "count" => 59, + "season" => 0 + ], + "ans" => [ + -0.04081236490074001, + -0.7956756034401067, + 0.8364879683408469 + ] + ]) +]; +} + +const eps = 1e-5; + +function ensure($x) +{ + if (!$x) { + die(1); + } +} + +function main() +{ + $input = getInput(); + foreach ($input as $row) { + $predictions = kml_catboost_predict_ht_multi("catboost_multiclass_tutorial_ht_catnum", $row['features_map']); + ensure(count($predictions) == count($row['ans'])); + for ($i = 0; $i < count($predictions); $i++) { + ensure(abs($predictions[$i] - $row['ans'][$i]) < eps); + } + } +} +main(); diff --git a/tests/python/tests/kml/catboost/php/catboost_tiny_1float_1hot_10trees.php b/tests/python/tests/kml/catboost/php/catboost_tiny_1float_1hot_10trees.php new file mode 100644 index 0000000000..caf4125685 --- /dev/null +++ b/tests/python/tests/kml/catboost/php/catboost_tiny_1float_1hot_10trees.php @@ -0,0 +1,71 @@ + [ + 1 + ], + "cat_features" => [ + "a" + ], + "ans" => 0.7363564860973357 + ]), + shape([ + "float_features" => [ + 2 + ], + "cat_features" => [ + "a" + ], + "ans" => 1.4672203311538694 + ]), + shape([ + "float_features" => [ + 3 + ], + "cat_features" => [ + "a" + ], + "ans" => -2.276542165549472 + ]), + shape([ + "float_features" => [ + 3 + ], + "cat_features" => [ + "b" + ], + "ans" => -0.9489540606737137 + ]), + shape([ + "float_features" => [ + 5 + ], + "cat_features" => [ + "b" + ], + "ans" => -0.9489540606737137 + ]), +]; +} + +const eps = 1e-5; + +function ensure($x) +{ + if (!$x) { + die(1); + } +} + +function main() +{ + $input = getInput(); + foreach ($input as $row) { + $prediction = kml_catboost_predict_vectors("catboost_tiny_1float_1hot_10trees", $row['float_features'], $row['cat_features']); + ensure(abs($prediction - $row['ans']) < eps); + } +} +main(); diff --git a/tests/python/tests/kml/catboost/php/unrelated_to_inference.php b/tests/python/tests/kml/catboost/php/unrelated_to_inference.php new file mode 100644 index 0000000000..39c27fe96b --- /dev/null +++ b/tests/python/tests/kml/catboost/php/unrelated_to_inference.php @@ -0,0 +1,20 @@ +^0M-5fZ@q%q4n)5RwAcrdV+PA@H~ 1, "1" => 1), + array("0" => 2, "1" => 2), + array("0" => 100, "1" => 100), + array("0" => 4, "1" => -10), + array("0" => 5, "1" => -555), + ); + $expected = array( + 0.745589, + 0.437018, + 0.437018, + 0.156840, + 0.156840, + ); + + $ans = kml_xgboost_predict_matrix("xgb_tiny_ht_direct_int_keys_to_fvalue", $inputs); + + $length = count($expected); + ensure($length == count($ans)); + for ($i = 0; $i < $length; $i++) { + ensure(abs($expected[$i] - $ans[$i]) < eps); + } +} +main(); diff --git a/tests/python/tests/kml/xgboost/php/xgb_tiny_ht_remap_str_keys_to_fvalue.php b/tests/python/tests/kml/xgboost/php/xgb_tiny_ht_remap_str_keys_to_fvalue.php new file mode 100644 index 0000000000..4cf1e822b4 --- /dev/null +++ b/tests/python/tests/kml/xgboost/php/xgb_tiny_ht_remap_str_keys_to_fvalue.php @@ -0,0 +1,36 @@ + 1, "weight" => 1, "useless_feature" => 0.1), + array("height" => 2, "weight" => 2, "useless_feature" => 0.12), + array("height" => 100, "weight" => 100, "useless_feature" => 0.123), + array("height" => 4, "weight" => -10, "useless_feature" => 0.1234), + array("height" => 5, "weight" => -555, "useless_feature" => 0.12345), + ); + $expected = array( + 0.745589, + 0.437018, + 0.437018, + 0.156840, + 0.156840, + ); + + $ans = kml_xgboost_predict_matrix("xgb_tiny_ht_remap_str_keys_to_fvalue", $inputs); + + $length = count($expected); + ensure($length == count($ans)); + for ($i = 0; $i < $length; $i++) { + ensure(abs($expected[$i] - $ans[$i]) < eps); + } +} +main(); From 8a04ac264c30ae1b76013709a5704a954c90ad93 Mon Sep 17 00:00:00 2001 From: Aleksandr Kirsanov Date: Tue, 23 Apr 2024 11:32:14 +0300 Subject: [PATCH 20/89] Add detailed documentation to kphp_ml.h (the same as PR description) --- runtime/kphp_ml/kphp_ml.h | 160 +++++++++++++++++++++++++++++++++++++- 1 file changed, 159 insertions(+), 1 deletion(-) diff --git a/runtime/kphp_ml/kphp_ml.h b/runtime/kphp_ml/kphp_ml.h index 55eab69578..18f35eba89 100644 --- a/runtime/kphp_ml/kphp_ml.h +++ b/runtime/kphp_ml/kphp_ml.h @@ -7,7 +7,165 @@ // They are almost identical, besides include paths and input types (`array` vs `unordered_map`). /* -todo detailed documentation +# About .kml files and kphp_ml in general + +KML means "KPHP ML", since it was invented for KPHP and VK.com. +KML unites xgboost and catboost (prediction only, not learning). +KML models are stored in files with *.kml* extension. + +KML is several times faster compared to native xgboost +and almost identical compared to native catboost. + +A final structure integrated into KPHP consists of the following: +1) custom xgboost implementation +2) custom catboost implementation +3) .kml files reader +4) buffers and kml models storage related to master-worker specifics +5) api to be called from PHP code + +To use ML from PHP code, call any function from `kphp_ml_interface.h` (KPHP only). +In plain PHP, there are no polyfills, and they are not planned to be implemented. + +# About "ml_experiments" private vkcom repo + +The code in the `kphp_ml` namespace is a final, production solution. + +While development, we tested lots of various implementations (both for xgboost/catboost) +in order to find an optimal one — they are located in the `ml_experiments` repository. + +All in all, `ml_experiments` repo contains: +1) lots of C++ implementations of algorithms that behave exactly like xgboost/catboost +2) tooling for testing and benchmarking them +3) ML models to be tested and benchmarked (some of them are from real production) +4) python scripts for learning and converting models +5) a converter to .kml + +Note, that some files exist both in KPHP and `ml_experiments`. +They are almost identical, besides include paths and input types (`array` vs `unordered_map`). +In the future development, they should be maintained synchronized. + +# Application-specific information in kml + +When a learned model is exported to xgboost .model file or catboost .cbm file, +it **does not** contain enough information to be evaluated. +Some information exists only at the moment of learning and thus must also be saved +along with xgboost/catboost exported models. + +For example, a prediction might need calibration (`*MULT+BIAS` or `log`) **AFTER** xgboost calculation. + +For example, input `[1234 => 0.98]` (feature_id #1234) must be remapped before passing to xgboost, +because this feature was #42 while training, but a valid input is #1234. +Hence, `[1234 => 42]` exists in reindex map. + +For example, some models were trained without zero values, and zeroes in input must be excluded. + +Ideally, an input should always contain correct indexes and shouldn't contain zeroes it the last case, +but in practice in VK.com, inputs are collected universally, and later applied to some model. +That's why one and the same input is remapped by model1 in a way 1, and by model2 in its own way. + +As a conclusion, training scripts must export not only xgboost/catboost models, but a .json file +with additional properties also — for converting to .kml and evaluating. +See `KmlPropertiesInJsonFile` in `ml_experiments`. + +.kml files, on the contrary, already contain all additional information inside, +because exporting to kml requires all that stuff. + +# InputKind + +Ideally, backend code must collect input that should be passed to a model directly. +For example, if a model was trained with features #1...#100, +an input could look like `[ 70 => 1.0, 23 => 7.42, ... ]`. + +But in practice and due to historical reasons, vkcom backend collects input in a different way, +and it can't be passed directly. It needs some transformations. Available types of input +and its transformation is `enum InputKind`, see below. + +# KML inference speed compared to xgboost/catboost + +Benchmarking shows, that a final KML predictor works 3–10 times faster compared to native xgboost. + +This is explained by several reasons and optimizations: +* compressed size of a tree node (8 bytes only) +* coordinates remapping +* better cache locality +* input vectorization and avoiding `if`s in code + +Remember, that KPHP workers are single-threaded, that's why it's compared with xgboost working +on a single thread, no GPU. + +.kml files are much more lightweight than .model xgboost files, since nodes are compressed +and all learning info is omitted. They can be loaded into memory very quickly, almost as POD bytes reading. + +When it comes to catboost, KML implementation is almost identical to +[native](https://github.com/catboost/catboost/blob/master/catboost/libs/model/model_export/resources). +But .kml files containing catboost models are also smaller than original .cbm files. + +# KPHP-specific implementation restrictions + +After PHP code is compiled to a server binary, it's launched as a pre-fork server. + +The master process loads all .kml files from the folder (provided as a cmd line option). +Note, that storage of models (and data of every model itself) is read-only, +that's why it's not copied to every process, and we are allowed to use `std` containers there. + +After fork, when PHP script is executed by every worker, it executes prediction, providing an input (PHP `array`). + +KPHP internals should be very careful of using std containers inside workers, since they allocate in heap, +which generally is bad because of signals handling. That's why KML evaluation doesn't use heap at all, +but when it needs memory for performing calculations, it uses pre-allocated `mutable_buffer`. +That mutable buffer is allocated once at every worker process start up, +its size is `max(calculate_mutable_buffer_size(i))`. Hence, it can fit any model calculation. + +A disappointing fact is that KPHP `array` is quite slow compared to `std::unordered_map`, +that's why a native C++ implementation is faster than a KPHP one +when an algorithm needs to iterate over input hashtables. + +# Looking backward: a brief history of ML in VK.com + +Historically, ML infrastructure in production was quite weird: ML models were tons of .php files +with autogenerated PHP code of decision trees, like +```php +function some_long_model_name_xxx_score_tree_3(array $x) { + if ($x[1657] < 0.00926230289) { + if ($x[1703] < 0.00839830097) { + if ($x[1657] < 0.00389328809) { + if ($x[1656] < 0.00147941126) { + return -0.216939136;} + return -0.215985224;} + ... +} +``` + +Hundreds of .php files, with hundreds of functions within each, with lots of lines `if else if else` +accessing input hashtables, sometimes transformed into vectors. + +That autogenerated code was placed in a separate repository, compiled with KPHP `-M lib`, and linked +into `vkcom` binary upon final compilation. The amount of models was so huge, that they took about 600 MB +of 1.5 GB production binary. The speed of inference, nevertheless, was quite fast, +especially when hashtables were transformed to vectors in advance. + +Time passed, and we decided to rewrite ML infrastructure from scratch. The goal was to +1) Get rid of codegenerated PHP code at all. +2) Greatly speed up current production. +3) Support catboost and categorial features. + +Obviously, there were two possible directions: +1) Import native xgboost and catboost libraries into KPHP runtime and write some transformers + from PHP input to native calls; store .model and .cbm files which can be loaded and executed. +2) Write a custom ML prediction kernel that works exactly like native xgboost/catboost, + but (if possible) much faster and much more lightweight; implement some .kml file format storing ML models. + +As one may guess, we finally head the second way. + +# Looking forward: possible future enhancements + +For now, provided solution it more than enough and solves all problems we face nowadays. +In the future, the following points might be considered as areas of investigation. + +* Support embedded and text features in catboost. +* Support onnx kernel for neural networks (also a custom implementation, of course). +* Use something more effective than `std::unordered_map` for reindex maps. +* Implement a thread pool in KPHP and parallelize inputs; it's safe, since they are read only. */ #pragma once From cc78821922c0e624627fda123147b50936a710f9 Mon Sep 17 00:00:00 2001 From: Mikhail Kornaukhov <66915223+mkornaukhov03@users.noreply.github.com> Date: Mon, 13 May 2024 12:47:52 +0300 Subject: [PATCH 21/89] Replace static_assert with std::enable_if (#994) Replace static_assert with std::enable_if for VertexAdaptor --- compiler/data/vertex-adaptor.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler/data/vertex-adaptor.h b/compiler/data/vertex-adaptor.h index 1a02090312..76dfea5b49 100644 --- a/compiler/data/vertex-adaptor.h +++ b/compiler/data/vertex-adaptor.h @@ -39,15 +39,13 @@ class VertexAdaptor { explicit VertexAdaptor(vertex_inner *impl) noexcept : impl_(impl) {} - template + template> VertexAdaptor(const VertexAdaptor &from) noexcept : impl_(static_cast *>(from.impl_)) { - static_assert(op_type_is_base_of(Op, FromOp), "Strange cast to not base vertex"); } - template + template> VertexAdaptor &operator=(const VertexAdaptor &from) noexcept { - static_assert(op_type_is_base_of(Op, FromOp), "Strange assignment to not base vertex"); impl_ = static_cast *>(from.impl_); return *this; } From da64cfce86caf11a2d1daa58550f6499084d4915 Mon Sep 17 00:00:00 2001 From: Mikhail Kornaukhov <66915223+mkornaukhov03@users.noreply.github.com> Date: Mon, 13 May 2024 12:48:41 +0300 Subject: [PATCH 22/89] Fix TLS threads count (#991) Fix TLS threads count --- compiler/threading/tls.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compiler/threading/tls.h b/compiler/threading/tls.h index b0eaae08a8..0b0f2a83f8 100644 --- a/compiler/threading/tls.h +++ b/compiler/threading/tls.h @@ -29,10 +29,14 @@ struct TLS { char dummy[4096]; }; - TLSRaw arr[MAX_THREADS_COUNT + 1]; + // The thread with thread_id = 0 is the main thread in which the scheduler's master code is executed. + // Threads with thread_id values in the range [1, MAX_THREADS_CNT] can be used as worker threads. + // An additional thread with thread_id = MAX_THREADS_CNT + 1 can be used to work with the CppDestDirInitializer. + // Therefore, the system requires a total of MAX_THREADS_CNT + 2 threads to be available. + TLSRaw arr[MAX_THREADS_COUNT + 2]; TLSRaw *get_raw(int id) { - assert(0 <= id && id <= MAX_THREADS_COUNT); + assert(0 <= id && id <= 1 + MAX_THREADS_COUNT); return &arr[id]; } From 4afc18c70475f3a8f35da71d9f0b04948d247fa8 Mon Sep 17 00:00:00 2001 From: Leonid Date: Fri, 17 May 2024 11:12:45 +0300 Subject: [PATCH 23/89] Added GNUInstallDirs CMake module (#998) Replaces hardcoded installation paths with paths from GNUInstallDirs. This should help ensure cross-distribution builds, at least at the directory compatibility level. --- CMakeLists.txt | 1 + common/tl/tl.cmake | 2 +- common/tlo-parsing/tlo-parsing.cmake | 6 +++--- flex/flex.cmake | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7468e7fa4b..59e70955e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ include(CMakePrintHelpers) include(CheckCXXCompilerFlag) include(AddFileDependencies) include(FetchContent) +include(GNUInstallDirs) # Global includes must be before all other includes/add_subdirectories include(cmake/utils.cmake) diff --git a/common/tl/tl.cmake b/common/tl/tl.cmake index 7aa0dbf2bf..ba272f234d 100644 --- a/common/tl/tl.cmake +++ b/common/tl/tl.cmake @@ -4,7 +4,7 @@ include(${COMMON_DIR}/tl2php/tl2php.cmake) install(TARGETS tl-compiler tl2php COMPONENT TL_TOOLS - RUNTIME DESTINATION bin) + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) install(DIRECTORY ${COMMON_DIR}/tl-files COMPONENT TL_TOOLS diff --git a/common/tlo-parsing/tlo-parsing.cmake b/common/tlo-parsing/tlo-parsing.cmake index 8b4e6070a2..34017891a8 100644 --- a/common/tlo-parsing/tlo-parsing.cmake +++ b/common/tlo-parsing/tlo-parsing.cmake @@ -25,8 +25,8 @@ set_target_properties(tlo_parsing_static PROPERTIES install(TARGETS tlo_parsing_static COMPONENT TLO_PARSING_DEV - ARCHIVE DESTINATION lib - PUBLIC_HEADER DESTINATION usr/include/tlo-parsing) + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION /usr/${CMAKE_INSTALL_INCLUDEDIR}/tlo-parsing) vk_add_library(tlo_parsing_shared SHARED $) set_target_properties(tlo_parsing_shared PROPERTIES OUTPUT_NAME tlo_parsing) @@ -37,7 +37,7 @@ endif() install(TARGETS tlo_parsing_shared COMPONENT TLO_PARSING - LIBRARY DESTINATION lib) + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) set(CPACK_DEBIAN_TLO_PARSING_PACKAGE_NAME "libtlo-parsing") set(CPACK_DEBIAN_TLO_PARSING_DESCRIPTION "Library files for the tlo parsing") diff --git a/flex/flex.cmake b/flex/flex.cmake index dc04bc67c9..9aa9f58fc1 100644 --- a/flex/flex.cmake +++ b/flex/flex.cmake @@ -35,8 +35,8 @@ set_target_properties(flex_data_shared flex_data_static install(TARGETS flex_data_shared flex_data_static COMPONENT FLEX - LIBRARY DESTINATION usr/lib - ARCHIVE DESTINATION usr/lib) + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) set(CPACK_DEBIAN_FLEX_PACKAGE_BREAKS "engine-kphp-runtime (<< 20190917), php5-vkext (<< 20190917), php7-vkext (<< 20190917)") set(CPACK_DEBIAN_FLEX_PACKAGE_NAME "vk-flex-data") From b20aead98b0abbd7a639ef5a1e09d27bad98d0da Mon Sep 17 00:00:00 2001 From: Aleksandr Kirsanov Date: Tue, 18 Jun 2024 14:55:53 +0300 Subject: [PATCH 24/89] resolve classes from "export" methods in .modulite.yaml (#1017) --- compiler/pipes/collect-required-and-classes.cpp | 4 ++++ .../modulite/010_mod_unreachable/Utils010/.modulite.yaml | 1 + .../Utils010/UnreachableNs/BnotherUn010.php | 7 +++++++ 3 files changed, 12 insertions(+) create mode 100644 tests/phpt/modulite/010_mod_unreachable/Utils010/UnreachableNs/BnotherUn010.php diff --git a/compiler/pipes/collect-required-and-classes.cpp b/compiler/pipes/collect-required-and-classes.cpp index 78305f26b8..b3bbd8f5b4 100644 --- a/compiler/pipes/collect-required-and-classes.cpp +++ b/compiler/pipes/collect-required-and-classes.cpp @@ -264,6 +264,10 @@ class CollectRequiredPass final : public FunctionPassBase { if (seems_like_classname) { require_class(modulite->modulite_namespace + e); } + size_t pos_classmember = e.find("::"); + if (pos_classmember != std::string::npos) { + require_class(modulite->modulite_namespace + e.substr(0, pos_classmember)); + } } } } diff --git a/tests/phpt/modulite/010_mod_unreachable/Utils010/.modulite.yaml b/tests/phpt/modulite/010_mod_unreachable/Utils010/.modulite.yaml index 19e9fb8ad7..6f42071db8 100644 --- a/tests/phpt/modulite/010_mod_unreachable/Utils010/.modulite.yaml +++ b/tests/phpt/modulite/010_mod_unreachable/Utils010/.modulite.yaml @@ -5,6 +5,7 @@ export: - "Strings010" - "UnreachableClass010" - "UnreachableNs\\AnotherUn010" + - "UnreachableNs\\BnotherUn010::nothing()" require: diff --git a/tests/phpt/modulite/010_mod_unreachable/Utils010/UnreachableNs/BnotherUn010.php b/tests/phpt/modulite/010_mod_unreachable/Utils010/UnreachableNs/BnotherUn010.php new file mode 100644 index 0000000000..8e363aa37c --- /dev/null +++ b/tests/phpt/modulite/010_mod_unreachable/Utils010/UnreachableNs/BnotherUn010.php @@ -0,0 +1,7 @@ + Date: Tue, 18 Jun 2024 14:57:29 +0300 Subject: [PATCH 25/89] resolve classes from @param phpdoc (#1015) Purpose: when a class occurs only in @param and is actually never created, resolve it in CollectRequiredPass. Previously, @param weren't analyzed there, because phpdocs haven't been parsed up to that execution point, and I didn't want to perform parsing twice. --- .../pipes/collect-required-and-classes.cpp | 18 ++++++++++++++++-- tests/phpt/cl/030_resolve_from_phpdoc.php | 8 ++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/compiler/pipes/collect-required-and-classes.cpp b/compiler/pipes/collect-required-and-classes.cpp index b3bbd8f5b4..761f585828 100644 --- a/compiler/pipes/collect-required-and-classes.cpp +++ b/compiler/pipes/collect-required-and-classes.cpp @@ -126,9 +126,23 @@ class CollectRequiredPass final : public FunctionPassBase { } } - // Collect classes only from type hints in PHP code, as phpdocs @param/@return haven't been parsed up to this execution point. - // TODO: shall we fix this? + // Collect classes from @param and type hints inline void require_all_classes_from_func_declaration(FunctionPtr f) { + // note, that @param has't been parsed up to this execution point, so parsing is done twice + // (here and later in ParseAndApplyPhpdocF), it's very ugly, since parsing is quite a heavy operation + // ideally, types in phpdoc should be parsed once in gentree, but actually, + // vkcom has so much syntax-invalid phpdocs, that it's unavailable + // (that "invalid" functions aren't reachable in fact, they just exist in a dead codebase, + // so their phpdocs aren't analyzed later, but trying to parse them in gentree leads to 10k errors) + if (f->phpdoc && !f->is_lambda()) { + for (const PhpDocTag &tag : f->phpdoc->tags) { + if (tag.type == PhpDocType::param) { + if (auto tag_parsed = tag.value_as_type_and_var_name(current_function, current_function->genericTs)) { + require_all_classes_in_phpdoc_type(tag_parsed.type_hint); + } + } + } + } for (const auto &p: f->get_params()) { if (p.as()->type_hint) { require_all_classes_in_phpdoc_type(p.as()->type_hint); diff --git a/tests/phpt/cl/030_resolve_from_phpdoc.php b/tests/phpt/cl/030_resolve_from_phpdoc.php index 634b2da022..5b604b0544 100644 --- a/tests/phpt/cl/030_resolve_from_phpdoc.php +++ b/tests/phpt/cl/030_resolve_from_phpdoc.php @@ -26,6 +26,14 @@ function f2() { else echo "a null\n"; } +/** + * @param ?\Classes\Z3Infer $b + */ +function f3($b) { + if ($b) $b->thisHasInfer(1,2); +} + f1(null); f2(); +f3(null); (new BB)->f(); From 5a6ad0c94d1af764106ae2f8594582b7c893c4c9 Mon Sep 17 00:00:00 2001 From: Aleksandr Kirsanov Date: Tue, 18 Jun 2024 14:58:16 +0300 Subject: [PATCH 26/89] Fix invalid __virt_clone generation for abstract classes (#1013) --- compiler/data/class-data.cpp | 2 +- .../phpt/clone_keyword/010_clone_abstract.php | 99 +++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 tests/phpt/clone_keyword/010_clone_abstract.php diff --git a/compiler/data/class-data.cpp b/compiler/data/class-data.cpp index c7150ad642..e97b8a7b73 100644 --- a/compiler/data/class-data.cpp +++ b/compiler/data/class-data.cpp @@ -114,7 +114,7 @@ FunctionPtr ClassData::add_virt_clone() { std::string virt_clone_f_name = replace_backslashes(name) + "$$" + NAME_OF_VIRT_CLONE; auto param_list = VertexAdaptor::create(gen_param_this({})); - auto body = !modifiers.is_abstract() + auto body = !is_interface() ? VertexAdaptor::create(VertexAdaptor::create(clone_this)) : VertexAdaptor::create(); auto v_op_function = VertexAdaptor::create(param_list, body); diff --git a/tests/phpt/clone_keyword/010_clone_abstract.php b/tests/phpt/clone_keyword/010_clone_abstract.php new file mode 100644 index 0000000000..ebab15f126 --- /dev/null +++ b/tests/phpt/clone_keyword/010_clone_abstract.php @@ -0,0 +1,99 @@ +@ok +bonus_reward1 = $bonus_reward1; + $this->bonus_reward2 = $bonus_reward2; + } + + public function __clone() { + if ($this->bonus_reward1 !== null) { + $this->bonus_reward1 = clone $this->bonus_reward1; + } + if ($this->bonus_reward2 !== null) { + $this->bonus_reward2 = clone $this->bonus_reward2; + } + } +} + +abstract class Reward1 { + abstract function out(); +} + +interface Reward2 { + function out(); +} + +final class PackReward1 extends Reward1 { + public ?int $pack = null; + + function __construct(?int $pack) { $this->pack = $pack; } + function out() { echo $this->pack, "\n"; } +} + +final class PackReward2 implements Reward2 { + public ?int $pack = null; + + function __construct(?int $pack) { $this->pack = $pack; } + function out() { echo $this->pack, "\n"; } +} + +final class DiscountReward1 extends Reward1 { + public ?string $discount = null; + + function __construct(?string $discount) { $this->discount = $discount; } + function out() { echo $this->discount, "!\n"; } +} + +final class DiscountReward2 implements Reward2 { + public ?string $discount = null; + + function __construct(?string $discount) { $this->discount = $discount; } + function out() { echo $this->discount, "!\n"; } +} + +function test1() { + $add = clone (new PurchaseAdd(null, null)); + var_dump($add->bonus_reward1 === null); + var_dump($add->bonus_reward2 === null); +} + +function test2() { + $orig1 = new PackReward1(10); + $orig2 = new PackReward2(10); + $add = clone (new PurchaseAdd($orig1, $orig2)); + $orig1->pack = 20; + $orig2->pack = 20; + $orig1->out(); + $orig2->out(); + $add->bonus_reward1->out(); + $add->bonus_reward2->out(); +} + +function test3() { + $add = clone (new PurchaseAdd(new DiscountReward1('asdf'), new DiscountReward2('asdf'))); + $add->bonus_reward1->out(); + $add->bonus_reward2->out(); +} + +function clTest(Reward1 $r) { + $r2 = clone $r; + if ($r instanceof PackReward1) { + $r->pack = 20; + } + $r2->out(); + $r->out(); +} + +test1(); +test2(); +test3(); + +clTest(new PackReward1(10)); From 2f6fd591bae0d2ff7909ebebd1e063f778764bb2 Mon Sep 17 00:00:00 2001 From: Aleksandr Kirsanov Date: Tue, 18 Jun 2024 14:59:23 +0300 Subject: [PATCH 27/89] Shorten too long file names at code generation (#1012) In vkcom, we sometimes see "file name too long". This MR shrinks too long file names, replacing in exact places long namespaces with their hashes. --- compiler/pipes/code-gen.cpp | 39 ++++++++++++++++++++++++++++++++++++- compiler/pipes/code-gen.h | 1 + 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/compiler/pipes/code-gen.cpp b/compiler/pipes/code-gen.cpp index 16fce776ef..254535afd1 100644 --- a/compiler/pipes/code-gen.cpp +++ b/compiler/pipes/code-gen.cpp @@ -26,9 +26,9 @@ #include "compiler/compiler-core.h" #include "compiler/data/class-data.h" #include "compiler/data/function-data.h" +#include "compiler/data/generics-mixins.h" #include "compiler/data/lib-data.h" #include "compiler/data/src-file.h" -#include "compiler/function-pass.h" #include "compiler/inferring/public.h" #include "compiler/pipes/collect-forkable-types.h" #include "compiler/type-hint.h" @@ -153,6 +153,24 @@ void CodeGenF::on_finish(DataStream> &os) { void CodeGenF::prepare_generate_function(FunctionPtr func) { std::string file_name = func->name; + + // shorten very long file names: they were generated by long namespaces (full class names) + // provided solution is to replace "very\\long\\namespace\\classname" with "{namespace_hash}\\classname" + // note, that we do this only at the moment of codegen (not at the moment of function name generation), + // because original (long) function names are sometimes parsed to show proper compilation errors + // (for instance, from "baseclass$$method$$contextclass" static=contextclass is parsed) + if (file_name.size() > 100 && func->is_instantiation_of_generic_function()) { + for (const auto &[_, type_hint] : *func->instantiationTs) { + type_hint->traverse([&file_name, this](const TypeHint *child) { + if (const auto *as_instance = child->try_as()) { + file_name = shorten_occurence_of_class_in_file_name(as_instance->resolve(), file_name); + } + }); + } + } + if (file_name.size() > 100 && func->context_class && func->class_id != func->context_class) { + file_name = shorten_occurence_of_class_in_file_name(func->context_class, file_name); + } std::replace(file_name.begin(), file_name.end(), '$', '@'); func->header_name = file_name + ".h"; @@ -176,6 +194,25 @@ void CodeGenF::prepare_generate_function(FunctionPtr func) { } } +std::string CodeGenF::shorten_occurence_of_class_in_file_name(ClassPtr occuring_class, const std::string &file_name) { + size_t pos = occuring_class->name.rfind('\\'); + if (pos == std::string::npos) { + return file_name; + } + vk::string_view occuring_namespace = vk::string_view{occuring_class->name}.substr(0, pos); + vk::string_view occuring_local_name = vk::string_view{occuring_class->name}.substr(pos + 1); + + std::string occurence1 = replace_characters(occuring_class->name, '\\', '$'); + std::string occurence2 = replace_characters(occuring_class->name, '\\', '_'); + std::string short_occur = fmt_format("{:x}${}", static_cast(vk::std_hash(occuring_namespace)), occuring_local_name); + + std::string shortened = file_name; + shortened = vk::replace_all(shortened, occurence1, short_occur); + shortened = vk::replace_all(shortened, occurence2, short_occur); + // printf("shortened file_name:\n %s\n -> %s\n", file_name.c_str(), shortened.c_str()); + return shortened; +} + std::string CodeGenF::calc_subdir_for_function(FunctionPtr func) { // place __construct and __invoke of lambdas to a separate dir, like lambda classes are placed to cl_l/ if (func->is_lambda()) { diff --git a/compiler/pipes/code-gen.h b/compiler/pipes/code-gen.h index 06971555af..58aa425400 100644 --- a/compiler/pipes/code-gen.h +++ b/compiler/pipes/code-gen.h @@ -18,6 +18,7 @@ class CodeGenF final : public SyncPipeF Date: Tue, 18 Jun 2024 18:15:48 +0300 Subject: [PATCH 28/89] Codegen globals as a linear memory piece, not as C++ variables (#1011) Before this MR, all constants / globals were codegenerated as separate C++ variables. This MR splits constants and globals into different storages and pipelines and targets to fully avoid mutable state in codegenerated C++ code. The purpose of this approach is to be able to compile a script into .so, load it multiple times, and execute concurrently. --- builtin-functions/_functions.txt | 2 +- common/php-functions.h | 35 +- .../code-gen/const-globals-batched-mem.cpp | 339 ++++++++++++++++++ compiler/code-gen/const-globals-batched-mem.h | 162 +++++++++ compiler/code-gen/declarations.cpp | 31 +- compiler/code-gen/declarations.h | 2 - compiler/code-gen/files/const-vars-init.cpp | 171 +++++++++ compiler/code-gen/files/const-vars-init.h | 21 ++ compiler/code-gen/files/function-header.cpp | 10 +- compiler/code-gen/files/function-source.cpp | 21 +- compiler/code-gen/files/function-source.h | 1 - .../files/global-vars-memory-stats.cpp | 111 ++++++ .../code-gen/files/global-vars-memory-stats.h | 23 ++ compiler/code-gen/files/global-vars-reset.cpp | 98 +++++ compiler/code-gen/files/global-vars-reset.h | 22 ++ .../files/global_vars_memory_stats.cpp | 97 ----- .../code-gen/files/global_vars_memory_stats.h | 21 -- compiler/code-gen/files/init-scripts.cpp | 152 ++++---- compiler/code-gen/files/shape-keys.cpp | 6 +- compiler/code-gen/files/shape-keys.h | 1 - compiler/code-gen/files/vars-cpp.cpp | 204 ----------- compiler/code-gen/files/vars-cpp.h | 28 -- compiler/code-gen/files/vars-reset.cpp | 107 ------ compiler/code-gen/files/vars-reset.h | 25 -- compiler/code-gen/naming.h | 29 +- compiler/code-gen/raw-data.h | 29 +- compiler/code-gen/vertex-compiler.cpp | 72 +++- compiler/compiler-core.cpp | 109 ++++-- compiler/compiler-core.h | 34 +- compiler/compiler-settings.cpp | 18 +- compiler/compiler-settings.h | 4 - compiler/compiler.cmake | 10 +- compiler/data/class-members.cpp | 2 +- compiler/data/function-data.h | 1 + compiler/data/var-data.cpp | 22 +- compiler/data/var-data.h | 12 +- compiler/data/vars-collector.cpp | 55 --- compiler/data/vars-collector.h | 27 -- compiler/kphp2cpp.cpp | 2 - compiler/lambda-utils.cpp | 2 +- compiler/make/make.cpp | 15 +- compiler/name-gen.cpp | 28 -- compiler/name-gen.h | 6 - compiler/pipes/analyzer.cpp | 6 + compiler/pipes/calc-bad-vars.cpp | 2 +- compiler/pipes/calc-empty-functions.cpp | 9 +- compiler/pipes/calc-locations.cpp | 22 +- compiler/pipes/code-gen.cpp | 69 ++-- compiler/pipes/code-gen.h | 1 - compiler/pipes/collect-const-vars.cpp | 34 +- compiler/pipes/convert-sprintf-calls.cpp | 8 +- compiler/pipes/convert-sprintf-calls.h | 2 +- compiler/pipes/final-check.cpp | 32 +- compiler/pipes/gen-tree-postprocess.cpp | 8 +- compiler/pipes/generate-virtual-methods.cpp | 2 +- compiler/pipes/optimization.cpp | 2 +- compiler/pipes/register-variables.cpp | 2 +- .../pipes/remove-empty-function-calls.cpp | 2 +- compiler/threading/hash-table.h | 1 + compiler/vertex-util.cpp | 13 - compiler/vertex-util.h | 1 - .../kphp-vs-php/compiler-cmd-options.md | 4 - runtime/array_functions.cpp | 3 + runtime/interface.cpp | 186 +++++----- runtime/interface.h | 13 +- runtime/memory_usage.cpp | 14 + runtime/memory_usage.h | 10 +- runtime/mixed.cpp | 2 + runtime/php-script-globals.cpp | 44 +++ runtime/php-script-globals.h | 49 +++ runtime/regexp.cpp | 1 + runtime/runtime.cmake | 2 + runtime/string.cpp | 2 + server/php-engine.cpp | 3 +- server/php-init-scripts.cpp | 2 +- server/php-init-scripts.h | 14 +- server/php-runner.cpp | 6 +- tests/cpp/runtime/_runtime-tests-env.cpp | 9 +- tests/phpt/constants/010_arrow_access.php | 3 +- tests/phpt/dl/1043_some_globals.php | 291 +++++++++++++++ tests/python/tests/ffi/test_ffi.py | 2 +- .../php/lib_examples/example1/php/index.php | 7 + tests/python/tests/libs/php/lib_user.php | 2 + 83 files changed, 1910 insertions(+), 1142 deletions(-) create mode 100644 compiler/code-gen/const-globals-batched-mem.cpp create mode 100644 compiler/code-gen/const-globals-batched-mem.h create mode 100644 compiler/code-gen/files/const-vars-init.cpp create mode 100644 compiler/code-gen/files/const-vars-init.h create mode 100644 compiler/code-gen/files/global-vars-memory-stats.cpp create mode 100644 compiler/code-gen/files/global-vars-memory-stats.h create mode 100644 compiler/code-gen/files/global-vars-reset.cpp create mode 100644 compiler/code-gen/files/global-vars-reset.h delete mode 100644 compiler/code-gen/files/global_vars_memory_stats.cpp delete mode 100644 compiler/code-gen/files/global_vars_memory_stats.h delete mode 100644 compiler/code-gen/files/vars-cpp.cpp delete mode 100644 compiler/code-gen/files/vars-cpp.h delete mode 100644 compiler/code-gen/files/vars-reset.cpp delete mode 100644 compiler/code-gen/files/vars-reset.h delete mode 100644 compiler/data/vars-collector.cpp delete mode 100644 compiler/data/vars-collector.h create mode 100644 runtime/memory_usage.cpp create mode 100644 runtime/php-script-globals.cpp create mode 100644 runtime/php-script-globals.h create mode 100644 tests/phpt/dl/1043_some_globals.php diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index ef816fff19..54444f053b 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -202,7 +202,7 @@ function memory_get_allocations() ::: tuple(int, int); function estimate_memory_usage($value ::: any) ::: int; // to enable this function, set KPHP_ENABLE_GLOBAL_VARS_MEMORY_STATS=1 -function get_global_vars_memory_stats($lower_bound ::: int = 0) ::: int[]; +function get_global_vars_memory_stats(int $lower_bound = 0) ::: int[]; function get_net_time() ::: float; function get_script_time() ::: float; diff --git a/common/php-functions.h b/common/php-functions.h index a90de4d647..8a127b712e 100644 --- a/common/php-functions.h +++ b/common/php-functions.h @@ -33,6 +33,16 @@ constexpr int STRLEN_VOID = STRLEN_ERROR; constexpr int STRLEN_FUTURE = STRLEN_ERROR; constexpr int STRLEN_FUTURE_QUEUE = STRLEN_ERROR; +constexpr int SIZEOF_STRING = 8; +constexpr int SIZEOF_ARRAY_ANY = 8; +constexpr int SIZEOF_MIXED = 16; +constexpr int SIZEOF_INSTANCE_ANY = 8; +constexpr int SIZEOF_OPTIONAL = 8; +constexpr int SIZEOF_FUTURE = 8; +constexpr int SIZEOF_FUTURE_QUEUE = 8; +constexpr int SIZEOF_REGEXP = 48; +constexpr int SIZEOF_UNKNOWN = 1; + class ExtraRefCnt { public: enum extra_ref_cnt_value { @@ -232,31 +242,6 @@ bool php_try_to_int(const char *s, size_t l, int64_t *val) { return true; } -//returns len of raw string representation or -1 on error -inline int string_raw_len(int src_len) { - if (src_len < 0 || src_len >= (1 << 30) - 13) { - return -1; - } - - return src_len + 13; -} - -//returns len of raw string representation and writes it to dest or returns -1 on error -inline int string_raw(char *dest, int dest_len, const char *src, int src_len) { - int raw_len = string_raw_len(src_len); - if (raw_len == -1 || raw_len > dest_len) { - return -1; - } - int *dest_int = reinterpret_cast (dest); - dest_int[0] = src_len; - dest_int[1] = src_len; - dest_int[2] = ExtraRefCnt::for_global_const; - memcpy(dest + 3 * sizeof(int), src, src_len); - dest[3 * sizeof(int) + src_len] = '\0'; - - return raw_len; -} - template inline constexpr int three_way_comparison(const T &lhs, const T &rhs) { return lhs < rhs ? -1 : diff --git a/compiler/code-gen/const-globals-batched-mem.cpp b/compiler/code-gen/const-globals-batched-mem.cpp new file mode 100644 index 0000000000..b1f588c3b3 --- /dev/null +++ b/compiler/code-gen/const-globals-batched-mem.cpp @@ -0,0 +1,339 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "compiler/code-gen/const-globals-batched-mem.h" + +#include +#include + +#include "common/php-functions.h" + +#include "compiler/compiler-core.h" +#include "compiler/code-gen/common.h" +#include "compiler/code-gen/includes.h" +#include "compiler/data/var-data.h" +#include "compiler/inferring/public.h" + +// see const-globals-batched-mem.h for detailed comments of what's going on + +namespace { + +ConstantsBatchedMem constants_batched_mem; +GlobalsBatchedMem globals_batched_mem; + +int calc_sizeof_tuple_shape(const TypeData *type); + +[[gnu::always_inline]] inline int calc_sizeof_in_bytes_runtime(const TypeData *type) { + switch (type->get_real_ptype()) { + case tp_int: + return type->use_optional() ? SIZEOF_OPTIONAL + sizeof(int64_t) : sizeof(int64_t); + case tp_float: + return type->use_optional() ? SIZEOF_OPTIONAL + sizeof(double) : sizeof(double); + case tp_string: + return type->use_optional() ? SIZEOF_OPTIONAL + SIZEOF_STRING : SIZEOF_STRING; + case tp_array: + return type->use_optional() ? SIZEOF_OPTIONAL + SIZEOF_ARRAY_ANY : SIZEOF_ARRAY_ANY; + case tp_regexp: + kphp_assert(!type->use_optional()); + return SIZEOF_REGEXP; + case tp_Class: + kphp_assert(!type->use_optional()); + return SIZEOF_INSTANCE_ANY; + case tp_mixed: + kphp_assert(!type->use_optional()); + return SIZEOF_MIXED; + case tp_bool: + return type->use_optional() ? 2 : 1; + case tp_tuple: + case tp_shape: + return calc_sizeof_tuple_shape(type); + case tp_future: + return type->use_optional() ? SIZEOF_OPTIONAL + SIZEOF_FUTURE : SIZEOF_FUTURE; + case tp_future_queue: + return type->use_optional() ? SIZEOF_OPTIONAL + SIZEOF_FUTURE_QUEUE : SIZEOF_FUTURE_QUEUE; + case tp_any: + return SIZEOF_UNKNOWN; + default: + kphp_error(0, fmt_format("Unable to detect sizeof() for type = {}", type->as_human_readable())); + return 0; + } +} + +[[gnu::noinline]] int calc_sizeof_tuple_shape(const TypeData *type) { + kphp_assert(vk::any_of_equal(type->ptype(), tp_tuple, tp_shape)); + + int result = 0; + bool has_align_8bytes = false; + for (auto sub = type->lookup_begin(); sub != type->lookup_end(); ++sub) { + int sub_sizeof = calc_sizeof_in_bytes_runtime(sub->second); + if (sub_sizeof >= 8) { + has_align_8bytes = true; + result = (result + 7) & -8; + } + result += sub_sizeof; + } + if (has_align_8bytes) { + result = (result + 7) & -8; + } + return type->use_optional() + ? has_align_8bytes ? SIZEOF_OPTIONAL + result : 1 + result + : result; +} + +} // namespace + + +void ConstantsBatchedMem::inc_count_by_type(const TypeData *type) { + if (type->use_optional()) { + count_of_type_other++; + return; + } + switch (type->get_real_ptype()) { + case tp_string: + count_of_type_string++; + break; + case tp_regexp: + count_of_type_regexp++; + break; + case tp_array: + count_of_type_array++; + return; + case tp_mixed: + count_of_type_mixed++; + break; + case tp_Class: + count_of_type_instance++; + break; + default: + count_of_type_other++; + } +} + +int ConstantsBatchedMem::detect_constants_batch_count(int n_constants) { + // these values are heuristics (don't use integer division, to avoid changing buckets count frequently) + if (n_constants > 1200000) return 2048; + if (n_constants > 800000) return 1536; + if (n_constants > 500000) return 1024; + if (n_constants > 100000) return 512; + if (n_constants > 10000) return 256; + if (n_constants > 5000) return 128; + if (n_constants > 1000) return 32; + if (n_constants > 500) return 16; + if (n_constants > 100) return 4; + return 1; +} + +const ConstantsBatchedMem &ConstantsBatchedMem::prepare_mem_and_assign_offsets(const std::vector &all_constants) { + ConstantsBatchedMem &mem = constants_batched_mem; + + const int N_BATCHES = detect_constants_batch_count(all_constants.size()); + mem.batches.resize(N_BATCHES); + + for (VarPtr var : all_constants) { + int batch_idx = static_cast(vk::std_hash(var->name) % N_BATCHES); + var->batch_idx = batch_idx; + mem.batches[batch_idx].n_constants++; + if (var->dependency_level > mem.batches[batch_idx].max_dep_level) { + mem.batches[batch_idx].max_dep_level = var->dependency_level; + } + } + + for (int batch_idx = 0; batch_idx < N_BATCHES; ++batch_idx) { + mem.batches[batch_idx].batch_idx = batch_idx; + mem.batches[batch_idx].constants.reserve(mem.batches[batch_idx].n_constants); + } + + for (VarPtr var : all_constants) { + mem.batches[var->batch_idx].constants.emplace_back(var); + } + + for (OneBatchInfo &batch : mem.batches) { + // sort constants by name to make codegen stable + std::sort(batch.constants.begin(), batch.constants.end(), [](VarPtr c1, VarPtr c2) -> bool { + return c1->name.compare(c2->name) < 0; + }); + + for (VarPtr var : batch.constants) { + const TypeData *var_type = tinf::get_type(var); + mem.inc_count_by_type(var_type); // count stat to output it + } + + mem.total_count += batch.constants.size(); + } + + return mem; +} + +void GlobalsBatchedMem::inc_count_by_origin(VarPtr var) { + if (var->is_class_static_var()) { + count_of_static_fields++; + } else if (var->is_function_static_var()) { + count_of_function_statics++; + } else if (vk::string_view{var->name}.starts_with("d$")) { + count_of_nonconst_defines++; + } else if (vk::string_view{var->name}.ends_with("$called")) { + count_of_require_once++; + } else if (!var->is_builtin_runtime) { + count_of_php_global_scope++; + } +} + +int GlobalsBatchedMem::detect_globals_batch_count(int n_globals) { + if (n_globals > 10000) return 256; + if (n_globals > 5000) return 128; + if (n_globals > 1000) return 32; + if (n_globals > 500) return 16; + if (n_globals > 100) return 4; + if (n_globals > 50) return 2; + return 1; +} + +const GlobalsBatchedMem &GlobalsBatchedMem::prepare_mem_and_assign_offsets(const std::vector &all_globals) { + GlobalsBatchedMem &mem = globals_batched_mem; + + const int N_BATCHES = detect_globals_batch_count(all_globals.size()); + mem.batches.resize(N_BATCHES); + + for (VarPtr var : all_globals) { + int batch_idx = static_cast(vk::std_hash(var->name) % N_BATCHES); + var->batch_idx = batch_idx; + mem.batches[batch_idx].n_globals++; + } + + for (int batch_idx = 0; batch_idx < N_BATCHES; ++batch_idx) { + mem.batches[batch_idx].batch_idx = batch_idx; + mem.batches[batch_idx].globals.reserve(mem.batches[batch_idx].n_globals); + } + + for (VarPtr var : all_globals) { + mem.batches[var->batch_idx].globals.emplace_back(var); + } + + for (OneBatchInfo &batch : mem.batches) { + // sort variables by name to make codegen stable + // note, that all_globals contains also function static vars (explicitly added), + // and their names can duplicate or be equal to global vars; + // hence, also sort by holder_func (though global vars don't have holder_func, since there's no point of declaration) + std::sort(batch.globals.begin(), batch.globals.end(), [](VarPtr c1, VarPtr c2) -> bool { + int cmp_name = c1->name.compare(c2->name); + if (cmp_name < 0) { + return true; + } else if (cmp_name > 0) { + return false; + } else if (c1 == c2) { + return false; + } else { + if (!c1->holder_func) return true; + if (!c2->holder_func) return false; + return c1->holder_func->name.compare(c2->holder_func->name) < 0; + } + }); + + int offset = 0; + + for (VarPtr var : batch.globals) { + const TypeData *var_type = tinf::get_type(var); + int cur_sizeof = (calc_sizeof_in_bytes_runtime(var_type) + 7) & -8; // min 8 bytes per variable + + var->offset_in_linear_mem = mem.total_mem_size + offset; // it's continuous + offset += cur_sizeof; + mem.inc_count_by_origin(var); + } + + // leave "spaces" between batches for less incremental re-compilation: + // when PHP code changes (introducing a new global, for example), offsets will be shifted + // only inside one batch, but not throughout the whole project + // (with the exception, when a rounded batch size exceeds next 1KB) + // note, that we don't do this for constants: while globals memory is a single continuous piece, + // constants, on the contrary, are physically independent C++ variables + offset = (offset + 1023) & -1024; + + mem.total_mem_size += offset; + mem.total_count += batch.globals.size(); + } + + return mem; +} + +void ConstantsExternCollector::add_extern_from_var(VarPtr var) { + kphp_assert(var->is_constant()); + extern_constants.insert(var); +} + +void ConstantsExternCollector::add_extern_from_init_val(VertexPtr init_val) { + if (auto var = init_val.try_as()) { + add_extern_from_var(var->var_id); + } + for (VertexPtr child : *init_val) { + add_extern_from_init_val(child); + } +} + +void ConstantsExternCollector::compile(CodeGenerator &W) const { + for (VarPtr c : extern_constants) { + W << "extern " << type_out(tinf::get_type(c)) << " " << c->name << ";" << NL; + } +} + +void ConstantsMemAllocation::compile(CodeGenerator &W) const { + const ConstantsBatchedMem &mem = constants_batched_mem; + + W << "// total_count = " << mem.total_count << NL; + W << "// count(string) = " << mem.count_of_type_string << NL; + W << "// count(regexp) = " << mem.count_of_type_regexp << NL; + W << "// count(array) = " << mem.count_of_type_array << NL; + W << "// count(mixed) = " << mem.count_of_type_mixed << NL; + W << "// count(instance) = " << mem.count_of_type_instance << NL; + W << "// count(other) = " << mem.count_of_type_other << NL; + W << "// n_batches = " << mem.batches.size() << NL; +} + +void GlobalsMemAllocation::compile(CodeGenerator &W) const { + const GlobalsBatchedMem &mem = globals_batched_mem; + + W << "// total_mem_size = " << mem.total_mem_size << NL; + W << "// total_count = " << mem.total_count << NL; + W << "// count(static fields) = " << mem.count_of_static_fields << NL; + W << "// count(function statics) = " << mem.count_of_function_statics << NL; + W << "// count(nonconst defines) = " << mem.count_of_nonconst_defines << NL; + W << "// count(require_once) = " << mem.count_of_require_once << NL; + W << "// count(php global scope) = " << mem.count_of_php_global_scope << NL; + W << "// n_batches = " << mem.batches.size() << NL; + + if (!G->is_output_mode_lib()) { + W << "php_globals.once_alloc_linear_mem(" << mem.total_mem_size << ");" << NL; + } else { + W << "php_globals.once_alloc_linear_mem(\"" << G->settings().static_lib_name.get() << "\", " << mem.total_mem_size << ");" << NL; + } +} + +void PhpMutableGlobalsAssignCurrent::compile(CodeGenerator &W) const { + W << "PhpScriptMutableGlobals &php_globals = PhpScriptMutableGlobals::current();" << NL; +} + +void PhpMutableGlobalsDeclareInResumableClass::compile(CodeGenerator &W) const { + W << "PhpScriptMutableGlobals &php_globals;" << NL; +} + +void PhpMutableGlobalsAssignInResumableConstructor::compile(CodeGenerator &W) const { + W << "php_globals(PhpScriptMutableGlobals::current())"; +} + +void PhpMutableGlobalsRefArgument::compile(CodeGenerator &W) const { + W << "PhpScriptMutableGlobals &php_globals"; +} + +void PhpMutableGlobalsConstRefArgument::compile(CodeGenerator &W) const { + W << "const PhpScriptMutableGlobals &php_globals"; +} + +void GlobalVarInPhpGlobals::compile(CodeGenerator &W) const { + if (global_var->is_builtin_runtime) { + W << "php_globals.get_superglobals().v$" << global_var->name; + } else if (!G->is_output_mode_lib()) { + W << "(*reinterpret_cast<" << type_out(tinf::get_type(global_var)) << "*>(php_globals.mem()+" << global_var->offset_in_linear_mem << "))"; + } else { + W << "(*reinterpret_cast<" << type_out(tinf::get_type(global_var)) << "*>(php_globals.mem_for_lib(\"" << G->settings().static_lib_name.get() << "\")+" << global_var->offset_in_linear_mem << "))"; + } +} diff --git a/compiler/code-gen/const-globals-batched-mem.h b/compiler/code-gen/const-globals-batched-mem.h new file mode 100644 index 0000000000..67970e036e --- /dev/null +++ b/compiler/code-gen/const-globals-batched-mem.h @@ -0,0 +1,162 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "compiler/data/data_ptr.h" +#include "compiler/data/vertex-adaptor.h" + +class TypeData; +class CodeGenerator; + +// Here we put all auto-extracted constants (const strings, const arrays, etc.). +// They are initialized on master process start and therefore accessible from all workers for reading. +// Every const has a special refcount: if PHP code tries to mutate it, it'll be copied to script memory. +// Moreover, every constant has dependency_level (var-data.h). For instance, an array of strings +// should be initialized after all strings have been initialized. +// +// Since their count is huge, they are batched, and initialization is done (codegenerated) per-batch +// (more concrete, all level0 for every batch, then level1, etc.). +// +// When codegen starts, prepare_mem_and_assign_offsets() is called. +// It splits variables into batches and calculates necessary properties used in codegen later. +// Note, that finally each constant is represented as an independent C++ variable +// (whereas mutable globals, see below, are all placed in a single memory piece). +// We tested different ways to use the same approach for constants also. It works, but +// either leads to lots of incremental re-compilation or inconsistent compilation times in vkcom. +// +// See const-vars-init.cpp and collect-const-vars.cpp. +class ConstantsBatchedMem { +public: + struct OneBatchInfo { + int batch_idx; + int n_constants{0}; + std::vector constants; + int max_dep_level{0}; + }; + +private: + friend struct ConstantsMemAllocation; + + int count_of_type_string = 0; + int count_of_type_regexp = 0; + int count_of_type_array = 0; + int count_of_type_mixed = 0; + int count_of_type_instance = 0; + int count_of_type_other = 0; + + int total_count = 0; + + std::vector batches; + + void inc_count_by_type(const TypeData *type); + +public: + static int detect_constants_batch_count(int n_constants); + static const ConstantsBatchedMem &prepare_mem_and_assign_offsets(const std::vector &all_constants); + + const std::vector &get_batches() const { return batches; } + const OneBatchInfo &get_batch(uint64_t batch_hash) const { return batches.at(batch_hash); } +}; + +// While constants are initialized once in master process, mutable globals exists in each script +// (and are initialized on script start, placed in script memory). +// +// Opposed to constants, mutable globals are NOT C++ variables: instead, they all are placed +// in a single linear memory piece (char *), every var has offset (VarData::offset_in_linear_mem), +// and we use (&reinterpret_cast) to access a variable at offset (GlobalVarInPhpGlobals below). +// The purpose of this approach is to avoid mutable state at the moment of code generation, +// so that we could potentially compile a script into .so and load it multiple times. +// +// Note, that for correct offset calculation, the compiler must be aware of sizeof() of any possible type. +// If (earlier) a global inferred a type `std::tuple`, g++ determined its size. +// Now, we need to compute sizes and offsets at the moment of code generation, and to do it +// exactly the same as g++ would. See const-globals-batched-mem.cpp for implementation. +// +// Another thing to point is that we also split globals into batches, but leave "spaces" in linear memory: +// [batch1, ...(nothing, rounded up to 1KB), batch2, ...(nothing), ...] +// It's done to achieve less incremental re-compilation: when PHP code changes introducing a new global, +// offsets will be shifted only inside one batch, but not throughout the whole project. +// +// See globals-vars-reset.cpp and (runtime) php-script-globals.h. +class GlobalsBatchedMem { +public: + struct OneBatchInfo { + int batch_idx; + int n_globals{0}; + std::vector globals; + }; + +private: + friend struct GlobalsMemAllocation; + + int count_of_static_fields = 0; + int count_of_function_statics = 0; + int count_of_nonconst_defines = 0; + int count_of_require_once = 0; + int count_of_php_global_scope = 0; + + int total_count = 0; + int total_mem_size = 0; + + std::vector batches; + + void inc_count_by_origin(VarPtr var); + +public: + static int detect_globals_batch_count(int n_globals); + static const GlobalsBatchedMem &prepare_mem_and_assign_offsets(const std::vector &all_globals); + + const std::vector &get_batches() const { return batches; } + const OneBatchInfo &get_batch(int batch_idx) const { return batches.at(batch_idx); } +}; + +struct ConstantsExternCollector { + void add_extern_from_var(VarPtr var); + void add_extern_from_init_val(VertexPtr init_val); + + void compile(CodeGenerator &W) const; + +private: + std::set extern_constants; +}; + +struct ConstantsMemAllocation { + void compile(CodeGenerator &W) const; +}; + +struct GlobalsMemAllocation { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsAssignCurrent { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsDeclareInResumableClass { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsAssignInResumableConstructor { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsRefArgument { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsConstRefArgument { + void compile(CodeGenerator &W) const; +}; + +struct GlobalVarInPhpGlobals { + VarPtr global_var; + + explicit GlobalVarInPhpGlobals(VarPtr global_var) + : global_var(global_var) {} + + void compile(CodeGenerator &W) const; +}; diff --git a/compiler/code-gen/declarations.cpp b/compiler/code-gen/declarations.cpp index b046e2f750..6692a35e23 100644 --- a/compiler/code-gen/declarations.cpp +++ b/compiler/code-gen/declarations.cpp @@ -7,6 +7,7 @@ #include "common/algorithms/compare.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/files/json-encoder-tags.h" #include "compiler/code-gen/files/tl2cpp/tl2cpp-utils.h" #include "compiler/code-gen/includes.h" @@ -25,10 +26,6 @@ #include "compiler/inferring/type-data.h" #include "compiler/tl-classes.h" -VarDeclaration VarExternDeclaration(VarPtr var) { - return {var, true, false}; -} - VarDeclaration VarPlainDeclaration(VarPtr var) { return {var, false, false}; } @@ -42,10 +39,6 @@ VarDeclaration::VarDeclaration(VarPtr var, bool extern_flag, bool defval_flag) : void VarDeclaration::compile(CodeGenerator &W) const { const TypeData *type = tinf::get_type(var); - if (var->is_builtin_global()) { - W << CloseNamespace(); - } - kphp_assert(type->ptype() != tp_void); W << (extern_flag ? "extern " : "") << TypeName(type) << " " << VarName(var); @@ -64,10 +57,6 @@ void VarDeclaration::compile(CodeGenerator &W) const { "decltype(const_begin(" << VarName(var) << "))" << " " << VarName(var) << name << ";" << NL; } } - - if (var->is_builtin_global()) { - W << OpenNamespace(); - } } FunctionDeclaration::FunctionDeclaration(FunctionPtr function, bool in_header, gen_out_style style) : @@ -441,18 +430,6 @@ ClassDeclaration::ClassDeclaration(ClassPtr klass) : klass(klass) { } -void ClassDeclaration::declare_all_variables(VertexPtr vertex, CodeGenerator &W) const { - if (!vertex) { - return; - } - for (auto child: *vertex) { - declare_all_variables(child, W); - } - if (auto var = vertex.try_as()) { - W << VarExternDeclaration(var->var_id); - } -} - std::unique_ptr ClassDeclaration::detect_if_needs_tl_usings() const { if (tl2cpp::is_php_class_a_tl_constructor(klass) && !tl2cpp::is_php_class_a_tl_array_item(klass)) { const auto &scheme = G->get_tl_classes().get_scheme(); @@ -480,11 +457,13 @@ void ClassDeclaration::compile(CodeGenerator &W) const { tl_dep_usings->compile_dependencies(W); } - klass->members.for_each([&](const ClassMemberInstanceField &f) { + ConstantsExternCollector c_mem_extern; + klass->members.for_each([&c_mem_extern](const ClassMemberInstanceField &f) { if (f.var->init_val) { - declare_all_variables(f.var->init_val, W); + c_mem_extern.add_extern_from_init_val(f.var->init_val); } }); + W << c_mem_extern << NL; auto get_all_interfaces = [klass = this->klass] { auto transform_to_src_name = [](CodeGenerator &W, InterfacePtr i) { W << i->src_name.c_str(); }; diff --git a/compiler/code-gen/declarations.h b/compiler/code-gen/declarations.h index 5a30147c7b..d46adc9cd6 100644 --- a/compiler/code-gen/declarations.h +++ b/compiler/code-gen/declarations.h @@ -26,7 +26,6 @@ struct VarDeclaration { void compile(CodeGenerator &W) const; }; -VarDeclaration VarExternDeclaration(VarPtr var); VarDeclaration VarPlainDeclaration(VarPtr var); struct FunctionDeclaration { @@ -140,7 +139,6 @@ struct ClassDeclaration : CodeGenRootCmd { IncludesCollector compile_front_includes(CodeGenerator &W) const; void compile_back_includes(CodeGenerator &W, IncludesCollector &&front_includes) const; void compile_job_worker_shared_memory_piece_methods(CodeGenerator &W, bool compile_declaration_only = false) const; - void declare_all_variables(VertexPtr v, CodeGenerator &W) const; std::unique_ptr detect_if_needs_tl_usings() const; }; diff --git a/compiler/code-gen/files/const-vars-init.cpp b/compiler/code-gen/files/const-vars-init.cpp new file mode 100644 index 0000000000..51c4f2de0b --- /dev/null +++ b/compiler/code-gen/files/const-vars-init.cpp @@ -0,0 +1,171 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "compiler/code-gen/files/const-vars-init.h" + +#include "common/algorithms/hashes.h" + +#include "compiler/code-gen/const-globals-batched-mem.h" +#include "compiler/code-gen/declarations.h" +#include "compiler/code-gen/namespace.h" +#include "compiler/code-gen/raw-data.h" +#include "compiler/code-gen/vertex-compiler.h" +#include "compiler/data/function-data.h" +#include "compiler/data/src-file.h" +#include "compiler/inferring/public.h" + +struct InitConstVar { + VarPtr var; + explicit InitConstVar(VarPtr var) : var(var) {} + + void compile(CodeGenerator &W) const { + Location save_location = stage::get_location(); + + VertexPtr init_val = var->init_val; + if (init_val->type() == op_conv_regexp) { + const auto &location = init_val->get_location(); + kphp_assert(location.function && location.file); + W << var->name << ".init (" << var->init_val << ", " << RawString(location.function->name) << ", " + << RawString(location.file->relative_file_name + ':' + std::to_string(location.line)) + << ");" << NL; + } else { + W << var->name << " = " << var->init_val << ";" << NL; + } + + stage::set_location(save_location); + } +}; + + +static void compile_raw_array(CodeGenerator &W, const VarPtr &var, int shift) { + if (shift == -1) { + W << InitConstVar(var); + W << var->name << ".set_reference_counter_to(ExtraRefCnt::for_global_const);" << NL << NL; + return; + } + + W << var->name << ".assign_raw((char *) &raw_arrays[" << shift << "]);" << NL << NL; +} + +ConstVarsInit::ConstVarsInit(const ConstantsBatchedMem &all_constants_in_mem) + : all_constants_in_mem(all_constants_in_mem) {} + +void ConstVarsInit::compile_const_init_part(CodeGenerator &W, const ConstantsBatchedMem::OneBatchInfo &batch) { + DepLevelContainer const_raw_array_vars; + DepLevelContainer other_const_vars; + DepLevelContainer const_raw_string_vars; + + IncludesCollector includes; + ConstantsExternCollector c_mem_extern; + for (VarPtr var : batch.constants) { + if (!G->is_output_mode_lib()) { + includes.add_var_signature_depends(var); + includes.add_vertex_depends(var->init_val); + } + c_mem_extern.add_extern_from_init_val(var->init_val); + } + W << includes; + + W << OpenNamespace(); + for (VarPtr var : batch.constants) { + W << type_out(tinf::get_type(var)) << " " << var->name << ";" << NL; + } + W << c_mem_extern; + + for (VarPtr var : batch.constants) { + switch (var->init_val->type()) { + case op_string: + const_raw_string_vars.add(var); + break; + case op_array: + const_raw_array_vars.add(var); + break; + default: + other_const_vars.add(var); + break; + } + } + + std::vector str_values(const_raw_string_vars.size()); + std::transform(const_raw_string_vars.begin(), const_raw_string_vars.end(), + str_values.begin(), + [](VarPtr var) { return var->init_val.as()->str_val; }); + + const std::vector const_string_shifts = compile_raw_data(W, str_values); + const std::vector const_array_shifts = compile_arrays_raw_representation(const_raw_array_vars, W); + const size_t max_dep_level = std::max({const_raw_string_vars.max_dep_level(), const_raw_array_vars.max_dep_level(), other_const_vars.max_dep_level(), 1ul}); + + size_t str_idx = 0; + size_t arr_idx = 0; + for (size_t dep_level = 0; dep_level < max_dep_level; ++dep_level) { + const std::string func_name_i = fmt_format("const_init_level{}_file{}", dep_level, batch.batch_idx); + FunctionSignatureGenerator(W) << NL << "void " << func_name_i << "()" << BEGIN; + + for (VarPtr var : const_raw_string_vars.vars_by_dep_level(dep_level)) { + W << var->name << ".assign_raw (&raw[" << const_string_shifts[str_idx++] << "]);" << NL; + } + + for (VarPtr var : const_raw_array_vars.vars_by_dep_level(dep_level)) { + compile_raw_array(W, var, const_array_shifts[arr_idx++]); + } + + for (VarPtr var: other_const_vars.vars_by_dep_level(dep_level)) { + W << InitConstVar(var); + const TypeData *type_data = var->tinf_node.get_type(); + if (vk::any_of_equal(type_data->ptype(), tp_array, tp_mixed, tp_string)) { + W << var->name; + if (type_data->use_optional()) { + W << ".val()"; + } + W << ".set_reference_counter_to(ExtraRefCnt::for_global_const);" << NL; + } + } + + W << END << NL; + } + + W << CloseNamespace(); +} + +void ConstVarsInit::compile_const_init(CodeGenerator &W, const ConstantsBatchedMem &all_constants_in_mem) { + W << OpenNamespace(); + + W << NL; + + FunctionSignatureGenerator(W) << "void const_vars_init() " << BEGIN; + W << ConstantsMemAllocation() << NL; + + int very_max_dep_level = 0; + for (const auto &batch : all_constants_in_mem.get_batches()) { + very_max_dep_level = std::max(very_max_dep_level, batch.max_dep_level); + } + + for (int dep_level = 0; dep_level <= very_max_dep_level; ++dep_level) { + for (const auto &batch : all_constants_in_mem.get_batches()) { + if (dep_level <= batch.max_dep_level) { + const std::string func_name_i = fmt_format("const_init_level{}_file{}", dep_level, batch.batch_idx); + // function declaration + W << "void " << func_name_i << "();" << NL; + // function call + W << func_name_i << "();" << NL; + } + } + } + W << END; + W << CloseNamespace(); +} + +void ConstVarsInit::compile(CodeGenerator &W) const { + for (const auto &batch : all_constants_in_mem.get_batches()) { + W << OpenFile("c." + std::to_string(batch.batch_idx) + ".cpp", "o_const_init", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_const_init_part(W, batch); + W << CloseFile(); + } + + W << OpenFile("const_vars_init.cpp", "", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_const_init(W, all_constants_in_mem); + W << CloseFile(); +} diff --git a/compiler/code-gen/files/const-vars-init.h b/compiler/code-gen/files/const-vars-init.h new file mode 100644 index 0000000000..585e0a0916 --- /dev/null +++ b/compiler/code-gen/files/const-vars-init.h @@ -0,0 +1,21 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "compiler/code-gen/code-gen-root-cmd.h" +#include "compiler/code-gen/code-generator.h" +#include "compiler/code-gen/const-globals-batched-mem.h" + +struct ConstVarsInit : CodeGenRootCmd { + explicit ConstVarsInit(const ConstantsBatchedMem &all_constants_in_mem); + + void compile(CodeGenerator &W) const final; + + static void compile_const_init_part(CodeGenerator& W, const ConstantsBatchedMem::OneBatchInfo& batch); + static void compile_const_init(CodeGenerator &W, const ConstantsBatchedMem &all_constants_in_mem); + +private: + const ConstantsBatchedMem &all_constants_in_mem; +}; diff --git a/compiler/code-gen/files/function-header.cpp b/compiler/code-gen/files/function-header.cpp index edabe8cde5..8a17edc921 100644 --- a/compiler/code-gen/files/function-header.cpp +++ b/compiler/code-gen/files/function-header.cpp @@ -5,6 +5,7 @@ #include "compiler/code-gen/files/function-header.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/files/function-source.h" #include "compiler/code-gen/includes.h" @@ -25,9 +26,12 @@ void FunctionH::compile(CodeGenerator &W) const { W << includes; W << OpenNamespace(); - for (auto const_var : function->explicit_header_const_var_ids) { - W << VarExternDeclaration(const_var) << NL; + + ConstantsExternCollector c_mem_extern; + for (VarPtr var : function->explicit_header_const_var_ids) { + c_mem_extern.add_extern_from_var(var); } + W << c_mem_extern << NL; if (function->is_inline) { W << "inline "; @@ -53,9 +57,7 @@ void FunctionH::compile(CodeGenerator &W) const { W << includes; W << OpenNamespace(); - declare_global_vars(function, W); declare_const_vars(function, W); - declare_static_vars(function, W); W << UnlockComments(); W << function->root << NL; W << LockComments(); diff --git a/compiler/code-gen/files/function-source.cpp b/compiler/code-gen/files/function-source.cpp index d605f28c2e..a5afbcd157 100644 --- a/compiler/code-gen/files/function-source.cpp +++ b/compiler/code-gen/files/function-source.cpp @@ -5,6 +5,7 @@ #include "compiler/code-gen/files/function-source.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/includes.h" #include "compiler/code-gen/namespace.h" @@ -16,22 +17,12 @@ FunctionCpp::FunctionCpp(FunctionPtr function) : function(function) { } -void declare_global_vars(FunctionPtr function, CodeGenerator &W) { - for (auto global_var : function->global_var_ids) { - W << VarExternDeclaration(global_var) << NL; - } -} - void declare_const_vars(FunctionPtr function, CodeGenerator &W) { - for (auto const_var : function->explicit_const_var_ids) { - W << VarExternDeclaration(const_var) << NL; - } -} - -void declare_static_vars(FunctionPtr function, CodeGenerator &W) { - for (auto static_var : function->static_var_ids) { - W << VarExternDeclaration(static_var) << NL; + ConstantsExternCollector c_mem_extern; + for (VarPtr var : function->explicit_const_var_ids) { + c_mem_extern.add_extern_from_var(var); } + W << c_mem_extern << NL; } void FunctionCpp::compile(CodeGenerator &W) const { @@ -50,9 +41,7 @@ void FunctionCpp::compile(CodeGenerator &W) const { W << includes; W << OpenNamespace(); - declare_global_vars(function, W); declare_const_vars(function, W); - declare_static_vars(function, W); W << UnlockComments(); W << function->root << NL; diff --git a/compiler/code-gen/files/function-source.h b/compiler/code-gen/files/function-source.h index 72332f56cf..7e7062c9c7 100644 --- a/compiler/code-gen/files/function-source.h +++ b/compiler/code-gen/files/function-source.h @@ -16,4 +16,3 @@ struct FunctionCpp : CodeGenRootCmd { void declare_global_vars(FunctionPtr function, CodeGenerator &W); void declare_const_vars(FunctionPtr function, CodeGenerator &W); -void declare_static_vars(FunctionPtr function, CodeGenerator &W); diff --git a/compiler/code-gen/files/global-vars-memory-stats.cpp b/compiler/code-gen/files/global-vars-memory-stats.cpp new file mode 100644 index 0000000000..6e5b7a62c6 --- /dev/null +++ b/compiler/code-gen/files/global-vars-memory-stats.cpp @@ -0,0 +1,111 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "compiler/code-gen/files/global-vars-memory-stats.h" + +#include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" +#include "compiler/code-gen/declarations.h" +#include "compiler/code-gen/includes.h" +#include "compiler/code-gen/namespace.h" +#include "compiler/code-gen/raw-data.h" +#include "compiler/data/src-file.h" +#include "compiler/inferring/public.h" + +GlobalVarsMemoryStats::GlobalVarsMemoryStats(const std::vector &all_globals) { + for (VarPtr global_var : all_globals) { + bool is_primitive = vk::any_of_equal(tinf::get_type(global_var)->get_real_ptype(), tp_bool, tp_int, tp_float, tp_regexp, tp_any); + if (!is_primitive && !global_var->is_builtin_runtime) { + all_nonprimitive_globals.push_back(global_var); + } + } + // to make codegen stable (here we use operator < of VarPtr, see var-data.cpp) + std::sort(all_nonprimitive_globals.begin(), all_nonprimitive_globals.end()); +} + +void GlobalVarsMemoryStats::compile(CodeGenerator &W) const { + int total_count = static_cast(all_nonprimitive_globals.size()); + int parts_cnt = static_cast(std::ceil(static_cast(total_count) / N_GLOBALS_PER_FILE)); + + W << OpenFile("globals_memory_stats.cpp", "", false) + << ExternInclude(G->settings().runtime_headers.get()) + << OpenNamespace(); + + // we don't take libs into account here (don't call "global memory stats" for every lib), + // since we have to guarantee that libs were compiled with a necessary flag also + // (most likely, not, then C++ compilation will fail) + + FunctionSignatureGenerator(W) << "array " << getter_name_ << "(int64_t lower_bound, " + << PhpMutableGlobalsConstRefArgument() << ")" << BEGIN + << "array result;" << NL + << "result.reserve(" << total_count << ", false);" << NL << NL; + + for (int part_id = 0; part_id < parts_cnt; ++part_id) { + const std::string func_name_i = getter_name_ + std::to_string(part_id); + // function declaration + FunctionSignatureGenerator(W) << "void " << func_name_i << "(int64_t lower_bound, array &result, " << PhpMutableGlobalsConstRefArgument() << ")" << SemicolonAndNL(); + // function call + W << func_name_i << "(lower_bound, result, php_globals);" << NL << NL; + } + + W << "return result;" << NL << END + << CloseNamespace() + << CloseFile(); + + for (int part_id = 0; part_id < parts_cnt; ++part_id) { + int offset = part_id * N_GLOBALS_PER_FILE; + int count = std::min(static_cast(all_nonprimitive_globals.size()) - offset, N_GLOBALS_PER_FILE); + + W << OpenFile("globals_memory_stats." + std::to_string(part_id) + ".cpp", "o_globals_memory_stats", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_getter_part(W, part_id, all_nonprimitive_globals, offset, count); + W << CloseFile(); + } +} + +void GlobalVarsMemoryStats::compile_getter_part(CodeGenerator &W, int part_id, const std::vector &global_vars, int offset, int count) { + IncludesCollector includes; + std::vector var_names; + var_names.reserve(count); + for (int i = 0; i < count; ++i) { + VarPtr global_var = global_vars[offset + i]; + includes.add_var_signature_depends(global_var); + std::string var_name; + if (global_var->is_function_static_var()) { + var_name = global_var->holder_func->name + "::"; + } + var_name.append(global_var->as_human_readable()); + var_names.emplace_back(std::move(var_name)); + } + + W << includes << NL + << OpenNamespace(); + + const auto var_name_shifts = compile_raw_data(W, var_names); + W << NL; + + FunctionSignatureGenerator(W) << "static string get_raw_string(int raw_offset) " << BEGIN; + W << "string str;" << NL + << "str.assign_raw(&raw[raw_offset]);" << NL + << "return str;" << NL + << END << NL << NL; + + FunctionSignatureGenerator(W) << "void " << getter_name_ << part_id << "(int64_t lower_bound, array &result, " + << PhpMutableGlobalsConstRefArgument() << ")" << BEGIN; + + if (count) { + W << "int64_t estimation = 0;" << NL; + } + for (int i = 0; i < count; ++i) { + VarPtr global_var = global_vars[offset + i]; + W << "// " << var_names[i] << NL + << "estimation = f$estimate_memory_usage(" << GlobalVarInPhpGlobals(global_var) << ");" << NL + << "if (estimation > lower_bound) " << BEGIN + << "result.set_value(get_raw_string(" << var_name_shifts[i] << "), estimation);" << NL + << END << NL; + } + + W << END; + W << CloseNamespace(); +} diff --git a/compiler/code-gen/files/global-vars-memory-stats.h b/compiler/code-gen/files/global-vars-memory-stats.h new file mode 100644 index 0000000000..8c324b3fed --- /dev/null +++ b/compiler/code-gen/files/global-vars-memory-stats.h @@ -0,0 +1,23 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "compiler/code-gen/code-gen-root-cmd.h" +#include "compiler/code-gen/code-generator.h" +#include "compiler/data/data_ptr.h" + +struct GlobalVarsMemoryStats : CodeGenRootCmd { + explicit GlobalVarsMemoryStats(const std::vector &all_globals); + + void compile(CodeGenerator &W) const final; + +private: + static void compile_getter_part(CodeGenerator &W, int part_id, const std::vector &global_vars, int offset, int count); + + std::vector all_nonprimitive_globals; + + static constexpr const char *getter_name_ = "globals_memory_stats_impl"; // hardcoded in runtime, see f$get_global_vars_memory_stats() + static constexpr int N_GLOBALS_PER_FILE = 512; +}; diff --git a/compiler/code-gen/files/global-vars-reset.cpp b/compiler/code-gen/files/global-vars-reset.cpp new file mode 100644 index 0000000000..1f1fdf4092 --- /dev/null +++ b/compiler/code-gen/files/global-vars-reset.cpp @@ -0,0 +1,98 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "compiler/code-gen/files/global-vars-reset.h" + +#include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" +#include "compiler/code-gen/declarations.h" +#include "compiler/code-gen/namespace.h" +#include "compiler/code-gen/vertex-compiler.h" +#include "compiler/data/src-file.h" + +GlobalVarsReset::GlobalVarsReset(const GlobalsBatchedMem &all_globals_in_mem) + : all_globals_in_mem(all_globals_in_mem) {} + +void GlobalVarsReset::compile_globals_reset_part(CodeGenerator &W, const GlobalsBatchedMem::OneBatchInfo &batch) { + IncludesCollector includes; + for (VarPtr var : batch.globals) { + includes.add_var_signature_depends(var); + } + W << includes; + W << OpenNamespace(); + + W << NL; + ConstantsExternCollector c_mem_extern; + for (VarPtr var : batch.globals) { + if (var->init_val) { + c_mem_extern.add_extern_from_init_val(var->init_val); + } + } + W << c_mem_extern << NL; + + FunctionSignatureGenerator(W) << "void global_vars_reset_file" << batch.batch_idx << "(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + for (VarPtr var : batch.globals) { + if (var->is_builtin_runtime) { // they are manually reset in runtime sources + continue; + } + + // todo probably, inline hard_reset_var() body, since it uses new(&)? + W << "// " << var->as_human_readable() << NL; + W << "hard_reset_var(" << GlobalVarInPhpGlobals(var); + if (var->init_val) { + W << ", " << var->init_val; + } + W << ");" << NL; + } + + W << END; + W << NL; + W << CloseNamespace(); +} + +void GlobalVarsReset::compile_globals_reset(CodeGenerator &W, const GlobalsBatchedMem &all_globals_in_mem) { + W << OpenNamespace(); + FunctionSignatureGenerator(W) << "void global_vars_reset(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + + for (const auto &batch : all_globals_in_mem.get_batches()) { + const std::string func_name_i = "global_vars_reset_file" + std::to_string(batch.batch_idx); + // function declaration + W << "void " << func_name_i << "(" << PhpMutableGlobalsRefArgument() << ");" << NL; + // function call + W << func_name_i << "(php_globals);" << NL; + } + + W << END; + W << NL; + W << CloseNamespace(); +} + +void GlobalVarsReset::compile_globals_allocate(CodeGenerator &W) { + W << OpenNamespace(); + FunctionSignatureGenerator(W) << "void global_vars_allocate(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + + W << GlobalsMemAllocation(); + + W << END << NL; + W << CloseNamespace(); +} + +void GlobalVarsReset::compile(CodeGenerator &W) const { + for (const auto &batch : all_globals_in_mem.get_batches()) { + W << OpenFile("globals_reset." + std::to_string(batch.batch_idx) + ".cpp", "o_globals_reset", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_globals_reset_part(W, batch); + W << CloseFile(); + } + + W << OpenFile("globals_reset.cpp", "", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_globals_reset(W, all_globals_in_mem); + W << CloseFile(); + + W << OpenFile("globals_allocate.cpp", "", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_globals_allocate(W); + W << CloseFile(); +} diff --git a/compiler/code-gen/files/global-vars-reset.h b/compiler/code-gen/files/global-vars-reset.h new file mode 100644 index 0000000000..0d519d8e35 --- /dev/null +++ b/compiler/code-gen/files/global-vars-reset.h @@ -0,0 +1,22 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "compiler/code-gen/code-gen-root-cmd.h" +#include "compiler/code-gen/code-generator.h" +#include "compiler/code-gen/const-globals-batched-mem.h" + +struct GlobalVarsReset : CodeGenRootCmd { + explicit GlobalVarsReset(const GlobalsBatchedMem &all_globals_in_mem); + + void compile(CodeGenerator &W) const final; + + static void compile_globals_reset_part(CodeGenerator &W, const GlobalsBatchedMem::OneBatchInfo &batch); + static void compile_globals_reset(CodeGenerator &W, const GlobalsBatchedMem &all_globals_in_mem); + static void compile_globals_allocate(CodeGenerator &W); + +private: + const GlobalsBatchedMem &all_globals_in_mem; +}; diff --git a/compiler/code-gen/files/global_vars_memory_stats.cpp b/compiler/code-gen/files/global_vars_memory_stats.cpp deleted file mode 100644 index d49a4e008e..0000000000 --- a/compiler/code-gen/files/global_vars_memory_stats.cpp +++ /dev/null @@ -1,97 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "compiler/code-gen/files/global_vars_memory_stats.h" - -#include "compiler/code-gen/common.h" -#include "compiler/code-gen/declarations.h" -#include "compiler/code-gen/includes.h" -#include "compiler/code-gen/namespace.h" -#include "compiler/code-gen/raw-data.h" -#include "compiler/data/src-file.h" -#include "compiler/data/vars-collector.h" -#include "compiler/inferring/public.h" - -GlobalVarsMemoryStats::GlobalVarsMemoryStats(SrcFilePtr main_file) : - main_file_{main_file} { -} - -void GlobalVarsMemoryStats::compile(CodeGenerator &W) const { - VarsCollector vars_collector{32, [](VarPtr global_var) { - return vk::none_of_equal(tinf::get_type(global_var)->get_real_ptype(), tp_bool, tp_int, tp_float, tp_any); - }}; - - vars_collector.collect_global_and_static_vars_from(main_file_->main_function); - - auto global_var_parts = vars_collector.flush(); - size_t global_vars_count = 0; - for (const auto &global_vars : global_var_parts) { - global_vars_count += global_vars.size(); - } - - W << OpenFile(getter_name_ + ".cpp", "", false) - << ExternInclude(G->settings().runtime_headers.get()) - << OpenNamespace(); - - FunctionSignatureGenerator(W) << "array " << getter_name_ << "(int64_t lower_bound) " << BEGIN - << "array result;" << NL - << "result.reserve(" << global_vars_count << ", false);" << NL << NL; - - for (size_t part_id = 0; part_id < global_var_parts.size(); ++part_id) { - W << "void " << getter_name_ << "_" << part_id << "(int64_t lower_bound, array &result);" << NL - << getter_name_ << "_" << part_id << "(lower_bound, result);" << NL << NL; - } - - W << "return result;" << NL << END - << CloseNamespace() - << CloseFile(); - - for (size_t part_id = 0; part_id < global_var_parts.size(); ++part_id) { - compile_getter_part(W, global_var_parts[part_id], part_id); - } -} - -void GlobalVarsMemoryStats::compile_getter_part(CodeGenerator &W, const std::set &global_vars, size_t part_id) const { - W << OpenFile(getter_name_ + "_" + std::to_string(part_id) + ".cpp", "o_" + getter_name_, false) - << ExternInclude(G->settings().runtime_headers.get()); - - IncludesCollector includes; - std::vector var_names; - var_names.reserve(global_vars.size()); - for (const auto &global_var : global_vars) { - includes.add_var_signature_depends(global_var); - std::string var_name; - if (global_var->is_function_static_var()) { - var_name = global_var->holder_func->name + "::"; - } - var_name.append(global_var->as_human_readable()); - var_names.emplace_back(std::move(var_name)); - } - - W << includes << NL - << OpenNamespace(); - - FunctionSignatureGenerator(W) << "static string get_raw_string(int raw_offset) " << BEGIN; - const auto var_name_shifts = compile_raw_data(W, var_names); - W << "string str;" << NL - << "str.assign_raw(&raw[raw_offset]);" << NL - << "return str;" << NL - << END << NL << NL; - - FunctionSignatureGenerator(W) << "void " << getter_name_ << "_" << part_id << "(int64_t lower_bound, array &result) " << BEGIN - << "int64_t estimation = 0;" << NL; - size_t var_num = 0; - for (auto global_var : global_vars) { - W << VarDeclaration(global_var, true, false) - << "estimation = f$estimate_memory_usage(" << VarName(global_var) << ");" << NL - << "if (estimation > lower_bound) " << BEGIN - << "result.set_value(get_raw_string(" << var_name_shifts[var_num++] << "), estimation);" << NL - << END << NL; - } - - W << END; - - W << CloseNamespace() - << CloseFile(); -} diff --git a/compiler/code-gen/files/global_vars_memory_stats.h b/compiler/code-gen/files/global_vars_memory_stats.h deleted file mode 100644 index a6f48d4da3..0000000000 --- a/compiler/code-gen/files/global_vars_memory_stats.h +++ /dev/null @@ -1,21 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include "compiler/code-gen/code-gen-root-cmd.h" -#include "compiler/code-gen/code-generator.h" -#include "compiler/data/data_ptr.h" - -struct GlobalVarsMemoryStats : CodeGenRootCmd { - explicit GlobalVarsMemoryStats(SrcFilePtr main_file); - - void compile(CodeGenerator &W) const final; - -private: - void compile_getter_part(CodeGenerator &W, const std::set &global_vars, size_t part_id) const; - - const std::string getter_name_{"get_global_vars_memory_stats_impl"}; - SrcFilePtr main_file_; -}; diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index 2fa40b65d5..a8d2d4fedb 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -5,6 +5,7 @@ #include "compiler/code-gen/files/init-scripts.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/files/shape-keys.h" #include "compiler/code-gen/includes.h" @@ -18,57 +19,49 @@ struct StaticInit { void compile(CodeGenerator &W) const; }; - void StaticInit::compile(CodeGenerator &W) const { - for (LibPtr lib: G->get_libs()) { + if (G->is_output_mode_lib()) { + return; + } + + // "const vars init" declarations + FunctionSignatureGenerator(W) << "void const_vars_init()" << SemicolonAndNL() << NL; + for (LibPtr lib : G->get_libs()) { if (lib && !lib->is_raw_php()) { W << OpenNamespace(lib->lib_namespace()); - FunctionSignatureGenerator(W) << "void global_init_lib_scripts()" << SemicolonAndNL(); + FunctionSignatureGenerator(W) << "void const_vars_init()" << SemicolonAndNL(); W << CloseNamespace(); } } - W << OpenNamespace(); - FunctionSignatureGenerator(W) << "void const_vars_init()" << SemicolonAndNL() << NL; - FunctionSignatureGenerator(W) << "void tl_str_const_init()" << SemicolonAndNL(); if (G->get_untyped_rpc_tl_used()) { FunctionSignatureGenerator(W) << "array gen$tl_fetch_wrapper(std::unique_ptr)" << SemicolonAndNL(); W << "extern array gen$tl_storers_ht;" << NL; FunctionSignatureGenerator(W) << "void fill_tl_storers_ht()" << SemicolonAndNL() << NL; } - if (G->settings().is_static_lib_mode()) { - FunctionSignatureGenerator(W) << "void global_init_lib_scripts() " << BEGIN; + FunctionSignatureGenerator(W) << ("const char *get_php_scripts_version()") << BEGIN << "return " << RawString(G->settings().php_code_version.get()) << ";" + << NL << END << NL << NL; + + FunctionSignatureGenerator(W) << ("char **get_runtime_options([[maybe_unused]] int *count)") << BEGIN; + const auto &runtime_opts = G->get_kphp_runtime_opts(); + if (runtime_opts.empty()) { + W << "return nullptr;" << NL; } else { - FunctionSignatureGenerator(W) << ("const char *get_php_scripts_version()") << BEGIN - << "return " << RawString(G->settings().php_code_version.get()) << ";" << NL - << END << NL << NL; - - FunctionSignatureGenerator(W) << ("char **get_runtime_options([[maybe_unused]] int *count)") << BEGIN; - const auto &runtime_opts = G->get_kphp_runtime_opts(); - if (runtime_opts.empty()) { - W << "return nullptr;" << NL; - } else { - W << "*count = " << runtime_opts.size() << ";" << NL; - for (size_t i = 0; i != runtime_opts.size(); ++i) { - W << "static char arg" << i << "[] = " << RawString{runtime_opts[i]} << ";" << NL; - } - W << "static char *argv[] = " << BEGIN; - for (size_t i = 0; i != runtime_opts.size(); ++i) { - W << "arg" << i << "," << NL; - } - W << END << ";" << NL - << "return argv;" << NL; + W << "*count = " << runtime_opts.size() << ";" << NL; + for (size_t i = 0; i != runtime_opts.size(); ++i) { + W << "static char arg" << i << "[] = " << RawString{runtime_opts[i]} << ";" << NL; } - W << END << NL << NL; - - FunctionSignatureGenerator(W) << ("void global_init_php_scripts() ") << BEGIN; - for (LibPtr lib: G->get_libs()) { - if (lib && !lib->is_raw_php()) { - W << lib->lib_namespace() << "::global_init_lib_scripts();" << NL; - } + W << "static char *argv[] = " << BEGIN; + for (size_t i = 0; i != runtime_opts.size(); ++i) { + W << "arg" << i << "," << NL; } + W << END << ";" << NL << "return argv;" << NL; } + W << END << NL << NL; + + FunctionSignatureGenerator(W) << ("void init_php_scripts_once_in_master() ") << BEGIN; + if (!G->settings().tl_schema_file.get().empty()) { W << "tl_str_const_init();" << NL; if (G->get_untyped_rpc_tl_used()) { @@ -77,11 +70,19 @@ void StaticInit::compile(CodeGenerator &W) const { } } W << "const_vars_init();" << NL; + for (LibPtr lib : G->get_libs()) { + if (lib && !lib->is_raw_php()) { + W << lib->lib_namespace() << "::const_vars_init();" << NL; + } + } + W << NL; + FunctionSignatureGenerator(W) << "void " << ShapeKeys::get_function_name() << "()" << SemicolonAndNL(); + W << ShapeKeys::get_function_name() << "();" << NL; const auto &ffi = G->get_ffi_root(); const auto &ffi_shared_libs = ffi.get_shared_libs(); if (!ffi_shared_libs.empty()) { - W << "ffi_env_instance = FFIEnv{" << ffi_shared_libs.size() << ", " << ffi.get_dynamic_symbols_num() << "};" << NL; + W << "ffi_env_instance = FFIEnv{" << ffi_shared_libs.size() << ", " << ffi.get_dynamic_symbols_num() << "};" << NL; W << "ffi_env_instance.funcs.dlopen = dlopen;" << NL; W << "ffi_env_instance.funcs.dlsym = dlsym;" << NL; for (const auto &lib : ffi_shared_libs) { @@ -99,7 +100,6 @@ void StaticInit::compile(CodeGenerator &W) const { } W << END << NL; - W << CloseNamespace(); } struct RunFunction { @@ -116,61 +116,58 @@ struct RunFunction { }; -struct GlobalResetFunction { - FunctionPtr function; - GlobalResetFunction(FunctionPtr function); +struct GlobalsResetFunction { + FunctionPtr main_function; + explicit GlobalsResetFunction(FunctionPtr main_function); void compile(CodeGenerator &W) const; }; -GlobalResetFunction::GlobalResetFunction(FunctionPtr function) : - function(function) { -} +GlobalsResetFunction::GlobalsResetFunction(FunctionPtr main_function) + : main_function(main_function) {} -void GlobalResetFunction::compile(CodeGenerator &W) const { +void GlobalsResetFunction::compile(CodeGenerator &W) const { + // "global vars reset" declarations + FunctionSignatureGenerator(W) << "void global_vars_allocate(" << PhpMutableGlobalsRefArgument() << ")" << SemicolonAndNL(); + FunctionSignatureGenerator(W) << "void global_vars_reset(" << PhpMutableGlobalsRefArgument() << ")" << SemicolonAndNL(); + W << NL; for (LibPtr lib: G->get_libs()) { if (lib && !lib->is_raw_php()) { W << OpenNamespace(lib->lib_namespace()); - FunctionSignatureGenerator(W) << "void lib_global_vars_reset()" << SemicolonAndNL(); + FunctionSignatureGenerator(W) << "void global_vars_allocate(" << PhpMutableGlobalsRefArgument() << ")" << SemicolonAndNL(); + FunctionSignatureGenerator(W) << "void global_vars_reset(" << PhpMutableGlobalsRefArgument() << ")" << SemicolonAndNL(); W << CloseNamespace(); } } - FunctionSignatureGenerator(W) << "void " << FunctionName(function) << "$global_reset() " << BEGIN; - W << "void " << GlobalVarsResetFuncName(function) << ";" << NL; - W << GlobalVarsResetFuncName(function) << ";" << NL; + // "global vars reset" calls + FunctionSignatureGenerator(W) << "void " << FunctionName(main_function) << "$globals_reset(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + W << "global_vars_reset(php_globals);" << NL; for (LibPtr lib: G->get_libs()) { if (lib && !lib->is_raw_php()) { - W << lib->lib_namespace() << "::lib_global_vars_reset();" << NL; + W << lib->lib_namespace() << "::global_vars_reset(php_globals);" << NL; } } W << END << NL; } -struct LibGlobalVarsReset { - const FunctionPtr &main_function; - LibGlobalVarsReset(const FunctionPtr &main_function); +struct LibRunFunction { + FunctionPtr main_function; + LibRunFunction(FunctionPtr main_function); void compile(CodeGenerator &W) const; }; -LibGlobalVarsReset::LibGlobalVarsReset(const FunctionPtr &main_function) : +LibRunFunction::LibRunFunction(FunctionPtr main_function) : main_function(main_function) { } -void LibGlobalVarsReset::compile(CodeGenerator &W) const { - W << OpenNamespace(); - FunctionSignatureGenerator(W) << "void lib_global_vars_reset() " << BEGIN - << "void " << GlobalVarsResetFuncName(main_function) << ";" << NL - << GlobalVarsResetFuncName(main_function) << ";" << NL - << END << NL << NL; - W << "extern bool v$" << main_function->file_id->get_main_func_run_var_name() << ";" << NL; - W << CloseNamespace(); - +void LibRunFunction::compile(CodeGenerator &W) const { + // "run" functions just calls the main file of a lib + // it's guaranteed that it doesn't contain code except declarations (body_value is empty), + // that's why we shouldn't deal with `if (!called)` global W << StaticLibraryRunGlobal(gen_out_style::cpp) << BEGIN << "using namespace " << G->get_global_namespace() << ";" << NL - << "if (!v$" << main_function->file_id->get_main_func_run_var_name() << ")" << BEGIN << FunctionName(main_function) << "();" << NL - << END << NL << END << NL << NL; } @@ -191,33 +188,32 @@ void InitScriptsCpp::compile(CodeGenerator &W) const { W << ExternInclude("dlfcn.h"); // dlopen, dlsym } - if (!G->settings().is_static_lib_mode()) { - W << NL; - FunctionSignatureGenerator(W) << "void global_init_php_scripts()" << SemicolonAndNL(); - FunctionSignatureGenerator(W) << "void init_php_scripts()" << SemicolonAndNL(); - } - W << NL << StaticInit() << NL; - if (G->settings().is_static_lib_mode()) { - W << LibGlobalVarsReset(main_file_id->main_function); + if (G->is_output_mode_lib()) { + W << LibRunFunction(main_file_id->main_function); W << CloseFile(); return; } W << RunFunction(main_file_id->main_function) << NL; - W << GlobalResetFunction(main_file_id->main_function) << NL; + W << GlobalsResetFunction(main_file_id->main_function) << NL; + + FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; - FunctionSignatureGenerator(W) << "void init_php_scripts() " << BEGIN; + W << "global_vars_allocate(php_globals);" << NL; + for (LibPtr lib: G->get_libs()) { + if (lib && !lib->is_raw_php()) { + W << lib->lib_namespace() << "::global_vars_allocate(php_globals);" << NL; + } + } - W << ShapeKeys::get_function_declaration() << ";" << NL; - W << ShapeKeys::get_function_name() << "();" << NL << NL; - W << FunctionName(main_file_id->main_function) << "$global_reset();" << NL; + W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL; W << "set_script (" << FunctionName(main_file_id->main_function) << "$run, " - << FunctionName(main_file_id->main_function) << "$global_reset);" << NL; + << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; W << END; @@ -234,7 +230,7 @@ void LibVersionHFile::compile(CodeGenerator &W) const { } void CppMainFile::compile(CodeGenerator &W) const { - kphp_assert(G->settings().is_server_mode() || G->settings().is_cli_mode()); + kphp_assert(G->is_output_mode_server() || G->is_output_mode_cli()); W << OpenFile("main.cpp"); W << ExternInclude("server/php-engine.h") << NL; diff --git a/compiler/code-gen/files/shape-keys.cpp b/compiler/code-gen/files/shape-keys.cpp index 1e72ae5fe1..18428273bf 100644 --- a/compiler/code-gen/files/shape-keys.cpp +++ b/compiler/code-gen/files/shape-keys.cpp @@ -14,15 +14,11 @@ std::string ShapeKeys::get_function_name() noexcept { return "init_shape_demangler"; } -std::string ShapeKeys::get_function_declaration() noexcept { - return "void " + get_function_name() + "()"; -} - void ShapeKeys::compile(CodeGenerator &W) const { W << OpenFile{"_shape_keys.cpp"}; W << ExternInclude{G->settings().runtime_headers.get()}; - FunctionSignatureGenerator{W} << NL << get_function_declaration() << BEGIN; + FunctionSignatureGenerator{W} << "void " << get_function_name() << "()" << BEGIN; W << "std::unordered_map shape_keys_storage{" << NL; for (const auto &[hash, key] : shape_keys_storage_) { diff --git a/compiler/code-gen/files/shape-keys.h b/compiler/code-gen/files/shape-keys.h index ab9fdaf383..585b1c32fb 100644 --- a/compiler/code-gen/files/shape-keys.h +++ b/compiler/code-gen/files/shape-keys.h @@ -18,7 +18,6 @@ struct ShapeKeys : CodeGenRootCmd { void compile(CodeGenerator &W) const final; static std::string get_function_name() noexcept; - static std::string get_function_declaration() noexcept; private: const std::map shape_keys_storage_; diff --git a/compiler/code-gen/files/vars-cpp.cpp b/compiler/code-gen/files/vars-cpp.cpp deleted file mode 100644 index e206c3b9ff..0000000000 --- a/compiler/code-gen/files/vars-cpp.cpp +++ /dev/null @@ -1,204 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "compiler/code-gen/files/vars-cpp.h" - -#include "common/algorithms/hashes.h" - -#include "compiler/code-gen/common.h" -#include "compiler/code-gen/declarations.h" -#include "compiler/code-gen/includes.h" -#include "compiler/code-gen/namespace.h" -#include "compiler/code-gen/raw-data.h" -#include "compiler/code-gen/vertex-compiler.h" -#include "compiler/data/src-file.h" -#include "compiler/data/var-data.h" -#include "compiler/stage.h" - -struct InitVar { - VarPtr var; - explicit InitVar(VarPtr var) : var(var) {} - - void compile(CodeGenerator &W) const { - Location save_location = stage::get_location(); - - VertexPtr init_val = var->init_val; - if (init_val->type() == op_conv_regexp) { - const auto &location = init_val->get_location(); - kphp_assert(location.function && location.file); - W << VarName(var) << ".init (" << var->init_val << ", " - << RawString(location.function->name) << ", " - << RawString(location.file->relative_file_name + ':' + std::to_string(location.line)) - << ");" << NL; - } else { - W << VarName(var) << " = " << var->init_val << ";" << NL; - } - - stage::set_location(save_location); - } -}; - - -static void add_dependent_declarations(VertexPtr vertex, std::set &dependent_vars) { - if (!vertex) { - return; - } - for (auto child: *vertex) { - add_dependent_declarations(child, dependent_vars); - } - if (auto var = vertex.try_as()) { - dependent_vars.emplace(var->var_id); - } -} - -void compile_raw_array(CodeGenerator &W, const VarPtr &var, int shift) { - if (shift == -1) { - W << InitVar(var); - W << VarName(var) << ".set_reference_counter_to(ExtraRefCnt::for_global_const);" << NL << NL; - return; - } - - W << VarName(var) << ".assign_raw((char *) &raw_arrays[" << shift << "]);" << NL << NL; -} - -static void compile_vars_part(CodeGenerator &W, const std::vector &vars, size_t part_id) { - std::string file_name = "vars" + std::to_string(part_id) + ".cpp"; - W << OpenFile(file_name, "o_vars_" + std::to_string(part_id / 100), false); - - W << ExternInclude(G->settings().runtime_headers.get()); - - DepLevelContainer const_raw_array_vars; - DepLevelContainer other_const_vars; - DepLevelContainer const_raw_string_vars; - std::set dependent_vars; - - IncludesCollector includes; - for (auto var : vars) { - if (!G->settings().is_static_lib_mode() || !var->is_builtin_global()) { - includes.add_var_signature_depends(var); - includes.add_vertex_depends(var->init_val); - } - } - W << includes; - - W << OpenNamespace(); - for (auto var : vars) { - if (G->settings().is_static_lib_mode() && var->is_builtin_global()) { - continue; - } - - W << VarDeclaration(var); - if (var->is_constant()) { - switch (var->init_val->type()) { - case op_string: - const_raw_string_vars.add(var); - break; - case op_array: - add_dependent_declarations(var->init_val, dependent_vars); - const_raw_array_vars.add(var); - break; - case op_var: - add_dependent_declarations(var->init_val, dependent_vars); - other_const_vars.add(var); - break; - default: - other_const_vars.add(var); - break; - } - } - } - - std::vector extern_depends; - std::set_difference(dependent_vars.begin(), dependent_vars.end(), - vars.begin(), vars.end(), std::back_inserter(extern_depends)); - for (auto var : extern_depends) { - W << VarExternDeclaration(var); - } - - std::vector values(const_raw_string_vars.size()); - std::transform(const_raw_string_vars.begin(), const_raw_string_vars.end(), - values.begin(), - [](const VarPtr &var){ return var->init_val.as()->get_string(); }); - auto const_string_shifts = compile_raw_data(W, values); - - const std::vector const_array_shifts = compile_arrays_raw_representation(const_raw_array_vars, W); - kphp_assert(const_array_shifts.size() == const_raw_array_vars.size()); - - - const size_t max_dep_level = std::max({const_raw_string_vars.max_dep_level(), const_raw_array_vars.max_dep_level(), other_const_vars.max_dep_level()}); - - size_t str_idx = 0; - size_t arr_idx = 0; - for (size_t dep_level = 0; dep_level < max_dep_level; ++dep_level) { - FunctionSignatureGenerator(W) << NL << "void const_vars_init_priority_" << dep_level << "_file_" << part_id << "()" << BEGIN; - - for (const auto &var : const_raw_string_vars.vars_by_dep_level(dep_level)) { - W << VarName(var) << ".assign_raw (&raw[" << const_string_shifts[str_idx++] << "]);" << NL; - } - - for (const auto &var : const_raw_array_vars.vars_by_dep_level(dep_level)) { - compile_raw_array(W, var, const_array_shifts[arr_idx++]); - } - - for (const auto &var: other_const_vars.vars_by_dep_level(dep_level)) { - W << InitVar(var); - const auto *type_data = var->tinf_node.get_type(); - PrimitiveType ptype = type_data->ptype(); - if (vk::any_of_equal(ptype, tp_array, tp_mixed, tp_string)) { - W << VarName(var); - if (type_data->use_optional()) { - W << ".val()"; - } - W << ".set_reference_counter_to(ExtraRefCnt::for_global_const);" << NL; - } - } - - W << END << NL; - } - - W << CloseNamespace(); - W << CloseFile(); -} - - -VarsCpp::VarsCpp(std::vector &&max_dep_levels, size_t parts_cnt) - : max_dep_levels_(std::move(max_dep_levels)) - , parts_cnt_(parts_cnt) { - kphp_assert(1 <= parts_cnt_); - kphp_error(parts_cnt_ <= G->settings().globals_split_count.get(), - fmt_format("Too many globals initialization .cpp files({}) for the specified globals_split_count({})", parts_cnt_, - G->settings().globals_split_count.get())); -} - -void VarsCpp::compile(CodeGenerator &W) const { - W << OpenFile("vars.cpp", "", false); - W << OpenNamespace(); - - FunctionSignatureGenerator(W) << "void const_vars_init() " << BEGIN; - - const int very_max_dep_level = *std::max_element(max_dep_levels_.begin(), max_dep_levels_.end()); - for (int dep_level = 0; dep_level <= very_max_dep_level; ++dep_level) { - for (size_t part_id = 0; part_id < parts_cnt_; ++part_id) { - if (dep_level <= max_dep_levels_[part_id]) { - auto init_fun = fmt_format("const_vars_init_priority_{}_file_{}();", dep_level, part_id); - // function declaration - W << "void " << init_fun << NL; - // function call - W << init_fun << NL; - } - } - } - W << END; - W << CloseNamespace(); - W << CloseFile(); -} - -VarsCppPart::VarsCppPart(std::vector &&vars_of_part, size_t part_id) - : vars_of_part_(std::move(vars_of_part)) - , part_id(part_id) {} - -void VarsCppPart::compile(CodeGenerator &W) const { - std::sort(vars_of_part_.begin(), vars_of_part_.end()); - compile_vars_part(W, vars_of_part_, part_id); -} diff --git a/compiler/code-gen/files/vars-cpp.h b/compiler/code-gen/files/vars-cpp.h deleted file mode 100644 index 6450761001..0000000000 --- a/compiler/code-gen/files/vars-cpp.h +++ /dev/null @@ -1,28 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include - -#include "compiler/code-gen/code-gen-root-cmd.h" -#include "compiler/code-gen/code-generator.h" - -struct VarsCpp : CodeGenRootCmd { - VarsCpp(std::vector &&max_dep_levels, size_t parts_cnt); - void compile(CodeGenerator &W) const final; - -private: - std::vector max_dep_levels_; - size_t parts_cnt_; -}; - -struct VarsCppPart : CodeGenRootCmd { - VarsCppPart(std::vector &&vars_of_part, size_t part_id); - void compile(CodeGenerator &W) const final; - -private: - mutable std::vector vars_of_part_; - size_t part_id; -}; diff --git a/compiler/code-gen/files/vars-reset.cpp b/compiler/code-gen/files/vars-reset.cpp deleted file mode 100644 index 3cd6f28e31..0000000000 --- a/compiler/code-gen/files/vars-reset.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "compiler/code-gen/files/vars-reset.h" - -#include "compiler/code-gen/common.h" -#include "compiler/code-gen/declarations.h" -#include "compiler/code-gen/includes.h" -#include "compiler/code-gen/namespace.h" -#include "compiler/code-gen/vertex-compiler.h" -#include "compiler/data/class-data.h" -#include "compiler/data/src-file.h" -#include "compiler/data/vars-collector.h" -#include "compiler/vertex.h" - -GlobalVarsReset::GlobalVarsReset(SrcFilePtr main_file) : - main_file_(main_file) { -} - -void GlobalVarsReset::declare_extern_for_init_val(VertexPtr v, std::set &externed_vars, CodeGenerator &W) { - if (auto var_vertex = v.try_as()) { - VarPtr var = var_vertex->var_id; - if (externed_vars.insert(var).second) { - W << VarExternDeclaration(var); - } - return; - } - for (VertexPtr son : *v) { - declare_extern_for_init_val(son, externed_vars, W); - } -} - -void GlobalVarsReset::compile_part(FunctionPtr func, const std::set &used_vars, int part_i, CodeGenerator &W) { - IncludesCollector includes; - for (auto var : used_vars) { - includes.add_var_signature_depends(var); - } - W << includes; - W << OpenNamespace(); - - std::set externed_vars; - for (auto var : used_vars) { - W << VarExternDeclaration(var); - if (var->init_val) { - declare_extern_for_init_val(var->init_val, externed_vars, W); - } - } - - FunctionSignatureGenerator(W) << "void " << GlobalVarsResetFuncName(func, part_i) << " " << BEGIN; - for (auto var : used_vars) { - if (G->settings().is_static_lib_mode() && var->is_builtin_global()) { - continue; - } - - W << "hard_reset_var(" << VarName(var); - //FIXME: brk and comments - if (var->init_val) { - W << ", " << var->init_val; - } - W << ");" << NL; - } - - W << END; - W << NL; - W << CloseNamespace(); -} - -void GlobalVarsReset::compile_func(FunctionPtr func, int parts_n, CodeGenerator &W) { - W << OpenNamespace(); - FunctionSignatureGenerator(W) << "void " << GlobalVarsResetFuncName(func) << " " << BEGIN; - - for (int i = 0; i < parts_n; i++) { - W << "void " << GlobalVarsResetFuncName(func, i) << ";" << NL; - W << GlobalVarsResetFuncName(func, i) << ";" << NL; - } - - W << END; - W << NL; - W << CloseNamespace(); -} - -void GlobalVarsReset::compile(CodeGenerator &W) const { - FunctionPtr main_func = main_file_->main_function; - - VarsCollector vars_collector{32}; - vars_collector.collect_global_and_static_vars_from(main_func); - auto used_vars = vars_collector.flush(); - - static const std::string vars_reset_src_prefix = "vars_reset."; - std::vector src_names(used_vars.size()); - for (int i = 0; i < used_vars.size(); i++) { - src_names[i] = vars_reset_src_prefix + std::to_string(i) + "." + main_func->src_name; - } - - for (int i = 0; i < used_vars.size(); i++) { - W << OpenFile(src_names[i], "o_vars_reset", false); - W << ExternInclude(G->settings().runtime_headers.get()); - compile_part(main_func, used_vars[i], i, W); - W << CloseFile(); - } - - W << OpenFile(vars_reset_src_prefix + main_func->src_name, "", false); - W << ExternInclude(G->settings().runtime_headers.get()); - compile_func(main_func, used_vars.size(), W); - W << CloseFile(); -} diff --git a/compiler/code-gen/files/vars-reset.h b/compiler/code-gen/files/vars-reset.h deleted file mode 100644 index 23387c9779..0000000000 --- a/compiler/code-gen/files/vars-reset.h +++ /dev/null @@ -1,25 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include "compiler/code-gen/code-gen-root-cmd.h" -#include "compiler/code-gen/code-generator.h" -#include "compiler/data/data_ptr.h" -#include "compiler/data/vertex-adaptor.h" - -struct GlobalVarsReset : CodeGenRootCmd { - explicit GlobalVarsReset(SrcFilePtr main_file); - - void compile(CodeGenerator &W) const final; - - static void compile_part(FunctionPtr func, const std::set &used_vars, int part_i, CodeGenerator &W); - - static void compile_func(FunctionPtr func, int parts_n, CodeGenerator &W); - - static void declare_extern_for_init_val(VertexPtr v, std::set &externed_vars, CodeGenerator &W); - -private: - SrcFilePtr main_file_; -}; diff --git a/compiler/code-gen/naming.h b/compiler/code-gen/naming.h index d4b8c77c8f..e17f9150f1 100644 --- a/compiler/code-gen/naming.h +++ b/compiler/code-gen/naming.h @@ -6,8 +6,6 @@ #include -#include "common/type_traits/list_of_types.h" - #include "compiler/code-gen/common.h" #include "compiler/code-gen/gen-out-style.h" #include "compiler/data/function-data.h" @@ -220,31 +218,8 @@ struct VarName { void compile(CodeGenerator &W) const { if (!name.empty()) { W << name; - return; - } - - if (var->is_function_static_var()) { - W << FunctionName(var->holder_func) << "$"; - } - - W << "v$" << var->name; - } -}; - -struct GlobalVarsResetFuncName { - explicit GlobalVarsResetFuncName(FunctionPtr main_func, int part = -1) : - main_func_(main_func), - part_(part) {} - - void compile(CodeGenerator &W) const { - W << FunctionName(main_func_) << "$global_vars_reset"; - if (part_ >= 0) { - W << std::to_string(part_); + } else { + W << "v$" << var->name; } - W << "()"; } - -private: - const FunctionPtr main_func_; - const int part_{-1}; }; diff --git a/compiler/code-gen/raw-data.h b/compiler/code-gen/raw-data.h index a4ba330460..1c1989708b 100644 --- a/compiler/code-gen/raw-data.h +++ b/compiler/code-gen/raw-data.h @@ -87,9 +87,32 @@ class RawString { std::vector compile_arrays_raw_representation(const DepLevelContainer &const_raw_array_vars, CodeGenerator &W); -template ().begin()), - typename = decltype(std::declval().end())> +//returns len of raw string representation or -1 on error +inline int string_raw_len(int src_len) { + if (src_len < 0 || src_len >= (1 << 30) - 13) { + return -1; + } + + return src_len + 13; +} + +//returns len of raw string representation and writes it to dest or returns -1 on error +inline int string_raw(char *dest, int dest_len, const char *src, int src_len) { + int raw_len = string_raw_len(src_len); + if (raw_len == -1 || raw_len > dest_len) { + return -1; + } + int *dest_int = reinterpret_cast (dest); + dest_int[0] = src_len; + dest_int[1] = src_len; + dest_int[2] = ExtraRefCnt::for_global_const; + memcpy(dest + 3 * sizeof(int), src, src_len); + dest[3 * sizeof(int) + src_len] = '\0'; + + return raw_len; +} + +template std::vector compile_raw_data(CodeGenerator &W, const Container &values) { std::string raw_data; std::vector const_string_shifts(values.size()); diff --git a/compiler/code-gen/vertex-compiler.cpp b/compiler/code-gen/vertex-compiler.cpp index 6f50d95b45..d75ce37105 100644 --- a/compiler/code-gen/vertex-compiler.cpp +++ b/compiler/code-gen/vertex-compiler.cpp @@ -11,6 +11,7 @@ #include "common/wrappers/likely.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/files/json-encoder-tags.h" #include "compiler/code-gen/files/tracing-autogen.h" @@ -1017,7 +1018,9 @@ void compile_foreach_ref_header(VertexAdaptor root, CodeGenerator &W BEGIN; - //save value + // save value: codegen `T &v$name = it.get_value()` + // note, that in global scope `v$name` remains a C++ variable (though other mutable globals are placed in linear mem) + // (there are also compile-time checks that foreach-ref global vars aren't used outside a loop) W << TypeName(tinf::get_type(x)) << " &"; W << x << " = " << it << ".get_value();" << NL; @@ -1345,13 +1348,16 @@ void compile_function_resumable(VertexAdaptor func_root, CodeGenera //MEMBER VARIABLES - for (auto var : func->param_ids) { + for (VarPtr var : func->param_ids) { kphp_error(!var->is_reference, "reference function parametrs are forbidden in resumable mode"); W << VarPlainDeclaration(var); } - for (auto var : func->local_var_ids) { + for (VarPtr var : func->local_var_ids) { W << VarPlainDeclaration(var); // inplace variables are stored as Resumable class fields as well } + if (func->has_global_vars_inside) { + W << PhpMutableGlobalsDeclareInResumableClass(); + } if (func->kphp_tracing) { // append KphpTracingFuncCallGuard as a member variable also ('start()' is called below) TracingAutogen::codegen_runtime_func_guard_declaration(W, func); } @@ -1363,19 +1369,34 @@ void compile_function_resumable(VertexAdaptor func_root, CodeGenera //CONSTRUCTOR FunctionSignatureGenerator(W) << FunctionClassName(func) << "(" << FunctionParams(func) << ")"; - if (!func->param_ids.empty()) { + bool has_members_in_constructor = !func->param_ids.empty() || !func->local_var_ids.empty() || func->has_global_vars_inside; + if (has_members_in_constructor) { + bool any_inited = false; W << " :" << NL << Indent(+2); - W << JoinValues(func->param_ids, ",", join_mode::multiple_lines, - [](CodeGenerator &W, VarPtr var) { - W << VarName(var) << "(" << VarName(var) << ")"; - }); + if (!func->param_ids.empty()) { + W << JoinValues(func->param_ids, ",", join_mode::multiple_lines, + [](CodeGenerator &W, VarPtr var) { + W << VarName(var) << "(" << VarName(var) << ")"; + }); + any_inited = true; + } if (!func->local_var_ids.empty()) { - W << "," << NL; + if (any_inited) { + W << "," << NL; + } + W << JoinValues(func->local_var_ids, ",", join_mode::multiple_lines, + [](CodeGenerator &W, VarPtr var) { + W << VarName(var) << "()"; + }); + any_inited = true; + } + if (func->has_global_vars_inside) { + if (any_inited) { + W << "," << NL; + } + W << PhpMutableGlobalsAssignInResumableConstructor(); + any_inited = true; } - W << JoinValues(func->local_var_ids, ",", join_mode::multiple_lines, - [](CodeGenerator &W, VarPtr var) { - W << VarName(var) << "()"; - }); W << Indent(-2); } W << " " << BEGIN << END << NL; @@ -1434,6 +1455,9 @@ void compile_function(VertexAdaptor func_root, CodeGenerator &W) { } W << FunctionDeclaration(func, false) << " " << BEGIN; + if (func->has_global_vars_inside) { + W << PhpMutableGlobalsAssignCurrent() << NL; + } if (func->kphp_tracing) { TracingAutogen::codegen_runtime_func_guard_declaration(W, func); @@ -1680,9 +1704,9 @@ bool try_compile_append_inplace(VertexAdaptor root, CodeGenerator &W // append all strings directly to the $lhs without creating a temporary // string for the $rhs result; also, grow $lhs accordingly, so it // can fit all the string parts - if (root->lhs()->type() == op_var) { - kphp_assert(tinf::get_type(root->lhs().as())->ptype() == tp_string); - compile_string_build_impl(root->rhs().as(), VarName{root->lhs().as()->var_id}, lhs_type, W); + if (auto as_var = root->lhs().try_as(); as_var && !as_var->var_id->is_in_global_scope()) { + kphp_assert(tinf::get_type(as_var)->ptype() == tp_string); + compile_string_build_impl(root->rhs().as(), VarName{as_var->var_id}, lhs_type, W); return true; } W << "(" << BEGIN; @@ -2131,9 +2155,21 @@ void compile_common_op(VertexPtr root, CodeGenerator &W) { case op_null: W << "Optional{}"; break; - case op_var: - W << VarName(root.as()->var_id); + case op_var: { + VarPtr var_id = root.as()->var_id; + if (var_id->is_constant()) { + // auto-extracted constant variables (const strings, arrays, etc.) in codegen are C++ variables + W << var_id->name; + } else if (var_id->is_in_global_scope() && !var_id->is_foreach_reference) { + // mutable globals, as opposed, are not C++ variables: instead, + // they all are placed in linear memory chunks, see php-script-globals.h + // with the only exception of `foreach (... as &$ref)` in global scope, see compile_foreach_ref_header() + W << GlobalVarInPhpGlobals(var_id); + } else { + W << VarName(var_id); + } break; + } case op_string: compile_string(root.as(), W); break; diff --git a/compiler/compiler-core.cpp b/compiler/compiler-core.cpp index 2be235feb0..014d04ec89 100644 --- a/compiler/compiler-core.cpp +++ b/compiler/compiler-core.cpp @@ -121,6 +121,14 @@ void CompilerCore::finish() { void CompilerCore::register_settings(CompilerSettings *settings) { kphp_assert (settings_ == nullptr); settings_ = settings; + + if (settings->mode.get() == "cli") { + output_mode = OutputMode::cli; + } else if (settings->mode.get() == "lib") { + output_mode = OutputMode::lib; + } else { + output_mode = OutputMode::server; + } } const CompilerSettings &CompilerCore::settings() const { @@ -197,6 +205,11 @@ FFIRoot &CompilerCore::get_ffi_root() { return ffi; } +OutputMode CompilerCore::get_output_mode() const { + return output_mode; +} + + vk::string_view CompilerCore::calc_relative_name(SrcFilePtr file, bool builtin) const { vk::string_view full_file_name = file->file_name; if (full_file_name.starts_with(settings_->base_dir.get())) { @@ -463,59 +476,72 @@ VarPtr CompilerCore::create_var(const std::string &name, VarData::Type type) { return var; } -VarPtr CompilerCore::get_global_var(const std::string &name, VarData::Type type, - VertexPtr init_val, bool *is_new_inserted) { - auto *node = global_vars_ht.at(vk::std_hash(name)); +VarPtr CompilerCore::get_global_var(const std::string &name, VertexPtr init_val) { + auto *node = globals_ht.at(vk::std_hash(name)); + + if (!node->data) { + AutoLocker locker(node); + if (!node->data) { + node->data = create_var(name, VarData::var_global_t); + node->data->init_val = init_val; + node->data->is_builtin_runtime = VarData::does_name_eq_any_builtin_runtime(name); + } + } + + return node->data; +} + +VarPtr CompilerCore::get_constant_var(const std::string &name, VertexPtr init_val, bool *is_new_inserted) { + auto *node = constants_ht.at(vk::std_hash(name)); VarPtr new_var; if (!node->data) { AutoLocker locker(node); if (!node->data) { - new_var = create_var(name, type); + new_var = create_var(name, VarData::var_const_t); new_var->init_val = init_val; node->data = new_var; } } - if (init_val) { + // when a string 'str' meets in several places in php code (same for [1,2,3] and other consts) + // it's created by a thread that first found it, and all others just ref to the same node + // here we make var->init_val->location stable, as it's sometimes used in code generation (locations of regexps, for example) + if (!new_var) { AutoLocker locker(node); - // to avoid of unstable locations, order them - if (node->data->init_val && node->data->init_val->get_location() < init_val->get_location()) { + if (node->data->init_val->get_location() < init_val->get_location()) { std::swap(node->data->init_val, init_val); } } - VarPtr var = node->data; if (is_new_inserted) { *is_new_inserted = static_cast(new_var); } + + VarPtr var = node->data; + // assert that one and the same init_val leads to one and the same var if (!new_var) { + kphp_assert(init_val); + kphp_assert(var->init_val->type() == init_val->type()); kphp_assert_msg(var->name == name, fmt_format("bug in compiler (hash collision) {} {}", var->name, name)); - if (init_val) { - kphp_assert(var->init_val->type() == init_val->type()); - switch (init_val->type()) { - case op_string: - kphp_assert(var->init_val->get_string() == init_val->get_string()); - break; - case op_conv_regexp: { - std::string &new_regexp = init_val.as()->expr().as()->str_val; - std::string &hashed_regexp = var->init_val.as()->expr().as()->str_val; - std::string msg = "hash collision: " + new_regexp + "; " + hashed_regexp; - - kphp_assert_msg(hashed_regexp == new_regexp, msg.c_str()); - break; - } - case op_array: { - std::string new_array_repr = VertexPtrFormatter::to_string(init_val); - std::string hashed_array_repr = VertexPtrFormatter::to_string(var->init_val); - - std::string msg = "hash collision: " + new_array_repr + "; " + hashed_array_repr; - - kphp_assert_msg(new_array_repr == hashed_array_repr, msg.c_str()); - break; - } - default: - break; + + switch (init_val->type()) { + case op_string: + kphp_assert(var->init_val->get_string() == init_val->get_string()); + break; + case op_conv_regexp: { + const std::string &new_regexp = init_val.as()->expr().as()->str_val; + const std::string &hashed_regexp = var->init_val.as()->expr().as()->str_val; + kphp_assert_msg(hashed_regexp == new_regexp, fmt_format("hash collision of regexp: {} vs {}", new_regexp, hashed_regexp)); + break; + } + case op_array: { + std::string new_array_repr = VertexPtrFormatter::to_string(init_val); + std::string hashed_array_repr = VertexPtrFormatter::to_string(var->init_val); + kphp_assert_msg(new_array_repr == hashed_array_repr, fmt_format("hash collision of arrays: {} vs {}", new_array_repr, hashed_array_repr)); + break; } + default: + break; } } return var; @@ -544,13 +570,22 @@ VarPtr CompilerCore::create_local_var(FunctionPtr function, const std::string &n } std::vector CompilerCore::get_global_vars() { - // static class variables are registered as globals, but if they're unused, - // then their types were never calculated; we don't need to export them to vars.cpp - return global_vars_ht.get_all_if([](VarPtr v) { - return v->tinf_node.was_recalc_finished_at_least_once(); + return globals_ht.get_all_if([](VarPtr v) { + // traits' static vars are added at the moment of parsing (class-members.cpp) + // but later never used, and tinf never executed for them + if (v->is_class_static_var() && v->class_id->is_trait()) { + return false; + } + // static vars for classes that are unused, are also present here + // probably, in the future, we'll detect unused globals and don't export them to C++ even as Unknown + return true; }); } +std::vector CompilerCore::get_constants_vars() { + return constants_ht.get_all(); +} + std::vector CompilerCore::get_classes() { return classes_ht.get_all(); } diff --git a/compiler/compiler-core.h b/compiler/compiler-core.h index 4ef7072f5a..944f0fec47 100644 --- a/compiler/compiler-core.h +++ b/compiler/compiler-core.h @@ -8,22 +8,27 @@ /*** Core ***/ //Consists mostly of functions that require synchronization -#include #include #include #include "common/algorithms/hashes.h" +#include "compiler/compiler-settings.h" +#include "compiler/composer.h" #include "compiler/data/data_ptr.h" #include "compiler/data/ffi-data.h" -#include "compiler/compiler-settings.h" +#include "compiler/function-colors.h" #include "compiler/index.h" #include "compiler/stats.h" #include "compiler/threading/data-stream.h" #include "compiler/threading/hash-table.h" #include "compiler/tl-classes.h" -#include "compiler/composer.h" -#include "compiler/function-colors.h" + +enum class OutputMode { + server, // -M server + cli, // -M cli + lib, // -M lib +}; class CompilerCore { private: @@ -33,13 +38,15 @@ class CompilerCore { TSHashTable functions_ht; TSHashTable classes_ht; TSHashTable defines_ht; - TSHashTable global_vars_ht; + TSHashTable constants_ht; // auto-collected constants (const strings / arrays / regexps / pure func calls); are inited once in a master process + TSHashTable globals_ht; // mutable globals (vars in global scope, class static fields); are reset for each php script inside worker processes TSHashTable libs_ht; TSHashTable modulites_ht; TSHashTable composer_json_ht; SrcFilePtr main_file; CompilerSettings *settings_; ComposerAutoloader composer_class_loader; + OutputMode output_mode; FFIRoot ffi; ClassPtr memcache_class; TlClasses tl_classes; @@ -67,6 +74,7 @@ class CompilerCore { SrcDirPtr register_dir(vk::string_view full_dir_name); FFIRoot &get_ffi_root(); + OutputMode get_output_mode() const; void register_main_file(const std::string &file_name, DataStream &os); SrcFilePtr require_file(const std::string &file_name, LibPtr owner_lib, DataStream &os, bool error_if_not_exists = true, bool builtin = false); @@ -103,11 +111,13 @@ class CompilerCore { DefinePtr get_define(std::string_view name); VarPtr create_var(const std::string &name, VarData::Type type); - VarPtr get_global_var(const std::string &name, VarData::Type type, VertexPtr init_val, bool *is_new_inserted = nullptr); + VarPtr get_global_var(const std::string &name, VertexPtr init_val); + VarPtr get_constant_var(const std::string &name, VertexPtr init_val, bool *is_new_inserted = nullptr); VarPtr create_local_var(FunctionPtr function, const std::string &name, VarData::Type type); SrcFilePtr get_main_file() { return main_file; } std::vector get_global_vars(); + std::vector get_constants_vars(); std::vector get_classes(); std::vector get_interfaces(); std::vector get_defines(); @@ -154,6 +164,18 @@ class CompilerCore { return is_functions_txt_parsed; } + bool is_output_mode_server() const { + return output_mode == OutputMode::server; + } + + bool is_output_mode_cli() const { + return output_mode == OutputMode::cli; + } + + bool is_output_mode_lib() const { + return output_mode == OutputMode::lib; + } + Stats stats; }; diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index df948b0d98..67560eeca1 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -204,18 +204,6 @@ void CompilerSettings::option_as_dir(KphpOption &path_option) noexc path_option.value_ = as_dir(path_option.value_); } -bool CompilerSettings::is_static_lib_mode() const { - return mode.get() == "lib"; -} - -bool CompilerSettings::is_server_mode() const { - return mode.get() == "server"; -} - -bool CompilerSettings::is_cli_mode() const { - return mode.get() == "cli"; -} - bool CompilerSettings::is_composer_enabled() const { return !composer_root.get().empty(); } @@ -230,7 +218,7 @@ void CompilerSettings::init() { runtime_sha256_file.value_ = get_full_path(runtime_sha256_file.get()); link_file.value_ = get_full_path(link_file.get()); - if (is_static_lib_mode()) { + if (mode.get() == "lib") { if (!tl_schema_file.get().empty()) { throw std::runtime_error{"Option " + tl_schema_file.get_env_var() + " is forbidden for static lib mode"}; } @@ -264,10 +252,6 @@ void CompilerSettings::init() { throw std::runtime_error{"Option " + threads_count.get_env_var() + " is expected to be <= " + std::to_string(MAX_THREADS_COUNT)}; } - if (globals_split_count.get() == 0) { - throw std::runtime_error{"globals-split-count may not be equal to zero"}; - } - for (std::string &include : includes.value_) { include = as_dir(include); } diff --git a/compiler/compiler-settings.h b/compiler/compiler-settings.h index 3a9e99a277..11342b99e7 100644 --- a/compiler/compiler-settings.h +++ b/compiler/compiler-settings.h @@ -128,7 +128,6 @@ class CompilerSettings : vk::not_copyable { KphpOption no_make; KphpOption jobs_count; KphpOption threads_count; - KphpOption globals_split_count; KphpOption require_functions_typing; KphpOption require_class_typing; @@ -185,9 +184,6 @@ class CompilerSettings : vk::not_copyable { KphpImplicitOption tl_classname_prefix; std::string get_version() const; - bool is_static_lib_mode() const; - bool is_server_mode() const; - bool is_cli_mode() const; bool is_composer_enabled() const; // reports whether composer compatibility mode is on color_settings get_color_settings() const; diff --git a/compiler/compiler.cmake b/compiler/compiler.cmake index 636aaaa6d6..6ae6aadadf 100644 --- a/compiler/compiler.cmake +++ b/compiler/compiler.cmake @@ -46,8 +46,7 @@ prepend(KPHP_COMPILER_DATA_SOURCES data/ src-dir.cpp src-file.cpp var-data.cpp - ffi-data.cpp - vars-collector.cpp) + ffi-data.cpp) prepend(KPHP_COMPILER_INFERRING_SOURCES inferring/ expr-node.cpp @@ -72,11 +71,14 @@ prepend(KPHP_COMPILER_INFERRING_SOURCES inferring/ prepend(KPHP_COMPILER_CODEGEN_SOURCES code-gen/ code-gen-task.cpp code-generator.cpp + const-globals-batched-mem.cpp declarations.cpp files/cmake-lists-txt.cpp + files/const-vars-init.cpp files/function-header.cpp files/function-source.cpp - files/global_vars_memory_stats.cpp + files/global-vars-memory-stats.cpp + files/global-vars-reset.cpp files/init-scripts.cpp files/json-encoder-tags.cpp files/lib-header.cpp @@ -91,8 +93,6 @@ prepend(KPHP_COMPILER_CODEGEN_SOURCES code-gen/ files/shape-keys.cpp files/tracing-autogen.cpp files/type-tagger.cpp - files/vars-cpp.cpp - files/vars-reset.cpp includes.cpp raw-data.cpp vertex-compiler.cpp diff --git a/compiler/data/class-members.cpp b/compiler/data/class-members.cpp index 16d006d7ae..434b62df58 100644 --- a/compiler/data/class-members.cpp +++ b/compiler/data/class-members.cpp @@ -58,7 +58,7 @@ inline ClassMemberStaticField::ClassMemberStaticField(ClassPtr klass, VertexAdap type_hint(type_hint) { std::string global_var_name = replace_backslashes(klass->name) + "$$" + root->get_string(); - var = G->get_global_var(global_var_name, VarData::var_global_t, def_val); + var = G->get_global_var(global_var_name, def_val); root->var_id = var; var->init_val = def_val; var->class_id = klass; diff --git a/compiler/data/function-data.h b/compiler/data/function-data.h index ed11dc03ca..bfe1139501 100644 --- a/compiler/data/function-data.h +++ b/compiler/data/function-data.h @@ -124,6 +124,7 @@ class FunctionData { bool has_lambdas_inside = false; // used for optimization after cloning (not to launch CloneNestedLambdasPass) bool has_var_tags_inside = false; // used for optimization (not to traverse body if no @var inside) bool has_commentTs_inside = false; // used for optimization (not to traverse body if no /*<...>*/ inside) + bool has_global_vars_inside = false; // used for codegeneration; true if function body refs any mutable php globals/superglobals (after cfg pass) bool warn_unused_result = false; bool is_flatten = false; bool is_pure = false; diff --git a/compiler/data/var-data.cpp b/compiler/data/var-data.cpp index 5aebcaaf27..6c7dc36915 100644 --- a/compiler/data/var-data.cpp +++ b/compiler/data/var-data.cpp @@ -32,13 +32,21 @@ const ClassMemberInstanceField *VarData::as_class_instance_field() const { return class_id->members.get_instance_field(name); } -// TODO Dirty HACK, should be removed -bool VarData::does_name_eq_any_builtin_global(const std::string &name) { - static const std::unordered_set names = { - "_SERVER", "_GET", "_POST", "_FILES", "_COOKIE", "_REQUEST", "_ENV", "argc", "argv", - "MC", "MC_True", "config", "Durov", "FullMCTime", "KPHP_MC_WRITE_STAT_PROBABILITY", - "d$PHP_SAPI"}; - return names.find(name) != names.end(); +bool VarData::does_name_eq_any_language_superglobal(const std::string &name) { + // these vars are 'superglobals' in PHP language: they are available in all scopes + static const std::unordered_set superglobal_names = { + "_SERVER", "_GET", "_POST", "_ENV", "_FILES", "_COOKIE", "_REQUEST", + }; + return superglobal_names.find(name) != superglobal_names.end(); +} + +bool VarData::does_name_eq_any_builtin_runtime(const std::string &name) { + // these vars are runtime built-ins, see PhpScriptBuiltInSuperGlobals + static const std::unordered_set runtime_names = { + "_SERVER", "_GET", "_POST", "_ENV", "_FILES", "_COOKIE", "_REQUEST", + "argc", "argv", "d$PHP_SAPI" + }; + return runtime_names.find(name) != runtime_names.end(); } bool operator<(VarPtr a, VarPtr b) { diff --git a/compiler/data/var-data.h b/compiler/data/var-data.h index c6d8572ea0..2da4b9de33 100644 --- a/compiler/data/var-data.h +++ b/compiler/data/var-data.h @@ -44,7 +44,10 @@ class VarData { bool marked_as_const = false; bool is_read_only = true; bool is_foreach_reference = false; - int dependency_level = 0; + bool is_builtin_runtime = false; // $_SERVER, $argv, etc., see PhpScriptBuiltInSuperGlobals in runtime + int dependency_level = 0; // for constants only (c_str$, c_arr$, etc) + int offset_in_linear_mem = -1; // for globals only (offset in g_linear_mem) + int batch_idx = -1; // for constants and globals, a number [0;N), see const-globals-batched-mem.h void set_uninited_flag(bool f); bool get_uninited_flag(); @@ -79,12 +82,9 @@ class VarData { return type_ == var_const_t; } - inline bool is_builtin_global() const { - return type_ == var_global_t && does_name_eq_any_builtin_global(name); - } - const ClassMemberStaticField *as_class_static_field() const; const ClassMemberInstanceField *as_class_instance_field() const; - static bool does_name_eq_any_builtin_global(const std::string &name); + static bool does_name_eq_any_language_superglobal(const std::string &name); + static bool does_name_eq_any_builtin_runtime(const std::string &name); }; diff --git a/compiler/data/vars-collector.cpp b/compiler/data/vars-collector.cpp deleted file mode 100644 index cafde30277..0000000000 --- a/compiler/data/vars-collector.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "compiler/data/vars-collector.h" - -#include "common/algorithms/hashes.h" - -#include "compiler/data/class-data.h" -#include "compiler/data/function-data.h" -#include "compiler/data/src-file.h" -#include "compiler/data/var-data.h" - -VarsCollector::VarsCollector(size_t parts, std::function vars_checker) : - collected_vars_(parts), - vars_checker_(std::move(vars_checker)) { -} - -void VarsCollector::collect_global_and_static_vars_from(FunctionPtr function) { - if (!visited_functions_.emplace(function).second) { - return; - } - - for (auto dep_function : function->dep) { - collect_global_and_static_vars_from(dep_function); - } - - add_vars(function->global_var_ids.begin(), function->global_var_ids.end()); - add_vars(function->static_var_ids.begin(), function->static_var_ids.end()); -} - -std::vector> VarsCollector::flush() { - visited_functions_.clear(); - - auto last_part = std::remove_if(collected_vars_.begin(), collected_vars_.end(), - [](const std::set &p) { return p.empty(); }); - collected_vars_.erase(last_part, collected_vars_.end()); - return std::move(collected_vars_); -} - -template -void VarsCollector::add_vars(It begin, It end) { - for (; begin != end; begin++) { - VarPtr var_id = *begin; - if (vars_checker_ && !vars_checker_(var_id)) { - continue; - } - const size_t var_hash = var_id->class_id ? - vk::std_hash(var_id->class_id->file_id->main_func_name) : - vk::std_hash(var_id->name); - - const size_t bucket = var_hash % collected_vars_.size(); - collected_vars_[bucket].emplace(var_id); - } -} diff --git a/compiler/data/vars-collector.h b/compiler/data/vars-collector.h deleted file mode 100644 index e97bacf320..0000000000 --- a/compiler/data/vars-collector.h +++ /dev/null @@ -1,27 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include -#include -#include - -#include "compiler/data/data_ptr.h" - -class VarsCollector { -public: - explicit VarsCollector(size_t parts, std::function vars_checker = {}); - - void collect_global_and_static_vars_from(FunctionPtr function); - std::vector> flush(); - -private: - template - void add_vars(It begin, It end); - - std::unordered_set visited_functions_; - std::vector> collected_vars_; - std::function vars_checker_; -}; diff --git a/compiler/kphp2cpp.cpp b/compiler/kphp2cpp.cpp index 48718d9fd3..94944e16d4 100644 --- a/compiler/kphp2cpp.cpp +++ b/compiler/kphp2cpp.cpp @@ -235,8 +235,6 @@ int main(int argc, char *argv[]) { 'j', "jobs-num", "KPHP_JOBS_COUNT", std::to_string(get_default_threads_count())); parser.add("Threads number for the transpilation", settings->threads_count, 't', "threads-count", "KPHP_THREADS_COUNT", std::to_string(get_default_threads_count())); - parser.add("Count of global variables per dedicated .cpp file. Lowering it could decrease compilation time", settings->globals_split_count, - "globals-split-count", "KPHP_GLOBALS_SPLIT_COUNT", "1536"); parser.add("Builtin tl schema. Incompatible with lib mode", settings->tl_schema_file, 'T', "tl-schema", "KPHP_TL_SCHEMA"); parser.add("Generate storers and fetchers for internal tl functions", settings->gen_tl_internals, diff --git a/compiler/lambda-utils.cpp b/compiler/lambda-utils.cpp index e654213301..093ebac69a 100644 --- a/compiler/lambda-utils.cpp +++ b/compiler/lambda-utils.cpp @@ -439,7 +439,7 @@ void auto_capture_vars_from_body_in_arrow_lambda(FunctionPtr f_lambda) { return var_name != "this" && // $this is captured by another approach, in non-arrow lambdas also var_name.find("::") == std::string::npos && var_name.find("$u") == std::string::npos && // not a superlocal var created in gentree - !VertexUtil::is_superglobal(var_name) && + !VarData::does_name_eq_any_language_superglobal(var_name) && !f_lambda->find_param_by_name(var_name) && !f_lambda->find_use_by_name(var_name); }; diff --git a/compiler/make/make.cpp b/compiler/make/make.cpp index 40206ea103..988d910585 100644 --- a/compiler/make/make.cpp +++ b/compiler/make/make.cpp @@ -388,7 +388,7 @@ static std::forward_list collect_imported_headers() { return imported_headers; } -static std::vector run_pre_make(const CompilerSettings &settings, FILE *make_stats_file, MakeSetup &make, Index &obj_index, File &bin_file) { +static std::vector run_pre_make(OutputMode output_mode, const CompilerSettings &settings, FILE *make_stats_file, MakeSetup &make, Index &obj_index, File &bin_file) { AutoProfiler profiler{get_profiler("Prepare Targets For Build")}; G->del_extra_files(); @@ -405,12 +405,13 @@ static std::vector run_pre_make(const CompilerSettings &settings, FILE * } auto lib_header_dirs = collect_imported_headers(); - return settings.is_static_lib_mode() ? kphp_make_static_lib_target(obj_index, G->get_index(), lib_header_dirs, make) - : kphp_make_target(obj_index, G->get_index(), lib_header_dirs, make); + return output_mode == OutputMode::lib ? kphp_make_static_lib_target(obj_index, G->get_index(), lib_header_dirs, make) + : kphp_make_target(obj_index, G->get_index(), lib_header_dirs, make); } void run_make() { const auto &settings = G->settings(); + OutputMode output_mode = G->get_output_mode(); FILE *make_stats_file = nullptr; if (!settings.stats_file.get().empty()) { make_stats_file = fopen(settings.stats_file.get().c_str(), "w"); @@ -425,10 +426,10 @@ void run_make() { kphp_assert(bin_file.read_stat() >= 0); MakeSetup make{make_stats_file, settings}; - auto objs = run_pre_make(settings, make_stats_file, make, obj_index, bin_file); + auto objs = run_pre_make(output_mode, settings, make_stats_file, make, obj_index, bin_file); stage::die_if_global_errors(); - if (settings.is_static_lib_mode()) { + if (output_mode == OutputMode::lib) { make.create_objs2static_lib_target(objs, &bin_file); } else { const std::string build_stage{"Compiling"}; @@ -440,7 +441,7 @@ void run_make() { } stage::die_if_global_errors(); - const std::string build_stage{settings.is_static_lib_mode() ? "Compiling" : "Linking"}; + const std::string build_stage{output_mode == OutputMode::lib ? "Compiling" : "Linking"}; AutoProfiler profiler{get_profiler(build_stage)}; bool ok = make.make_target(&bin_file, build_stage, settings.jobs_count.get()); @@ -459,7 +460,7 @@ void run_make() { if (!settings.user_binary_path.get().empty()) { hard_link_or_copy(bin_file.path, settings.user_binary_path.get()); } - if (settings.is_static_lib_mode()) { + if (output_mode == OutputMode::lib) { copy_static_lib_to_out_dir(std::move(bin_file)); } } diff --git a/compiler/name-gen.cpp b/compiler/name-gen.cpp index b41a788441..0538a26029 100644 --- a/compiler/name-gen.cpp +++ b/compiler/name-gen.cpp @@ -23,34 +23,6 @@ std::string gen_anonymous_function_name(FunctionPtr function) { return gen_unique_name("lambda", function); } -std::string gen_const_string_name(const std::string &str) { - return fmt_format("const_string$us{:x}", vk::std_hash(str)); -} - -std::string gen_const_regexp_name(const std::string &str) { - return fmt_format("const_regexp$us{:x}", vk::std_hash(str)); -} - -bool is_array_suitable_for_hashing(VertexPtr vertex) { - return vertex->type() == op_array && CheckConst::is_const(vertex); -} - -// checks that inlined as define' value constructor is suitable to be stored as constant var -bool is_object_suitable_for_hashing(VertexPtr vertex) { - return vertex->type() == op_define_val && vertex.as()->value()->type() == op_func_call - && vertex.as()->value()->extra_type == op_ex_constructor_call && vertex->const_type == cnst_const_val; -} - -std::string gen_const_object_name(const VertexAdaptor &def) { - kphp_assert_msg(def->value()->type() == op_func_call, "Internal error: expected op_define_val "); - auto obj_hash = ObjectHash::calc_hash(def); - return fmt_format("const_obj$us{:x}", obj_hash); -} - -std::string gen_const_array_name(const VertexAdaptor &array) { - return fmt_format("const_array$us{:x}", ArrayHash::calc_hash(array)); -} - std::string gen_unique_name(const std::string& prefix, FunctionPtr function) { if (!function) { function = stage::get_function(); diff --git a/compiler/name-gen.h b/compiler/name-gen.h index cf8ad75534..619b53fee9 100644 --- a/compiler/name-gen.h +++ b/compiler/name-gen.h @@ -12,12 +12,6 @@ std::string gen_anonymous_scope_name(FunctionPtr parent_function); std::string gen_anonymous_function_name(FunctionPtr parent_function); std::string gen_unique_name(const std::string& prefix, FunctionPtr function = FunctionPtr{}); -std::string gen_const_string_name(const std::string &str); -std::string gen_const_regexp_name(const std::string &str); -bool is_object_suitable_for_hashing(VertexPtr vertex); -std::string gen_const_object_name(const VertexAdaptor &array); -bool is_array_suitable_for_hashing(VertexPtr vertex); -std::string gen_const_array_name(const VertexAdaptor &array); std::string resolve_uses(FunctionPtr resolve_context, const std::string &class_name); ClassPtr resolve_class_of_arrow_access(FunctionPtr function, VertexPtr lhs, VertexPtr v); diff --git a/compiler/pipes/analyzer.cpp b/compiler/pipes/analyzer.cpp index 6405a7c776..11b60338a0 100644 --- a/compiler/pipes/analyzer.cpp +++ b/compiler/pipes/analyzer.cpp @@ -92,6 +92,12 @@ VertexPtr CommonAnalyzerPass::on_enter_vertex(VertexPtr vertex) { if (var->is_constant()) { run_function_pass(var->init_val, this); } + if (var->is_in_global_scope()) { + // save a flag, that a function's body accesses mutable global / static vars (to codegen `php_globals` variable) + // note, that assigning `has_global_vars_inside = !global_var_ids.empty()` is incorrect: + // for example if a function declares `global $v` but not uses it, or its uses are dropped off after cfg pass + current_function->has_global_vars_inside = true; + } return vertex; } if (vertex->rl_type == val_none) { diff --git a/compiler/pipes/calc-bad-vars.cpp b/compiler/pipes/calc-bad-vars.cpp index 7843108fc3..b539b17df4 100644 --- a/compiler/pipes/calc-bad-vars.cpp +++ b/compiler/pipes/calc-bad-vars.cpp @@ -588,7 +588,7 @@ class CalcBadVars { function->dep = std::move(call_graph.graph[function]); } - if (!G->settings().is_static_lib_mode()) { + if (!G->is_output_mode_lib()) { return; } diff --git a/compiler/pipes/calc-empty-functions.cpp b/compiler/pipes/calc-empty-functions.cpp index 461526a7d1..64314c9a0f 100644 --- a/compiler/pipes/calc-empty-functions.cpp +++ b/compiler/pipes/calc-empty-functions.cpp @@ -4,10 +4,9 @@ #include "compiler/pipes/calc-empty-functions.h" +#include "compiler/compiler-core.h" #include "compiler/data/function-data.h" #include "compiler/data/src-file.h" -#include "compiler/function-pass.h" -#include "compiler/vertex.h" namespace { @@ -49,6 +48,7 @@ FunctionData::body_value get_vertex_body_type(VertexPtr vertex) { case op_seq: return calc_seq_body_type(vertex.as()); case op_empty: + case op_global: return FunctionData::body_value::empty; default: return FunctionData::body_value::non_empty; @@ -89,5 +89,10 @@ FunctionData::body_value calc_function_body_type(FunctionPtr f) { void CalcEmptyFunctions::execute(FunctionPtr f, DataStream &os) { stage::set_function(f); f->body_seq = calc_function_body_type(f); + + if (f->is_main_function() && G->is_output_mode_lib() && G->get_main_file()->main_function == f) { + kphp_error(f->body_seq == FunctionData::body_value::empty, "main php file of a lib mustn't contain code, only declarations"); + } + os << f; } diff --git a/compiler/pipes/calc-locations.cpp b/compiler/pipes/calc-locations.cpp index e53828f384..89d34584c2 100644 --- a/compiler/pipes/calc-locations.cpp +++ b/compiler/pipes/calc-locations.cpp @@ -5,20 +5,24 @@ #include "compiler/pipes/calc-locations.h" #include "compiler/data/class-data.h" +#include "compiler/data/var-data.h" void CalcLocationsPass::on_start() { if (current_function->type == FunctionData::func_class_holder) { - current_function->class_id->members.for_each([](ClassMemberInstanceField &f) { - stage::set_line(f.root->location.line); - f.root->location = stage::get_location(); + current_function->class_id->members.for_each([this](ClassMemberInstanceField &f) { + f.root->location = Location{current_function->file_id, current_function, f.root->location.line}; + if (f.var->init_val) { + f.var->init_val.set_location_recursively(f.root->location); + } }); - current_function->class_id->members.for_each([](ClassMemberStaticField &f) { - stage::set_line(f.root->location.line); - f.root->location = stage::get_location(); + current_function->class_id->members.for_each([this](ClassMemberStaticField &f) { + f.root->location = Location{current_function->file_id, current_function, f.root->location.line}; + if (f.var->init_val) { + f.var->init_val.set_location_recursively(f.root->location); + } }); - current_function->class_id->members.for_each([](ClassMemberConstant &c) { - stage::set_line(c.value->location.line); - c.value.set_location(stage::get_location()); + current_function->class_id->members.for_each([this](ClassMemberConstant &c) { + c.value.set_location_recursively(Location{current_function->file_id, current_function, c.value->location.line}); }); } } diff --git a/compiler/pipes/code-gen.cpp b/compiler/pipes/code-gen.cpp index 254535afd1..500873ab66 100644 --- a/compiler/pipes/code-gen.cpp +++ b/compiler/pipes/code-gen.cpp @@ -4,26 +4,27 @@ #include "compiler/pipes/code-gen.h" -#include "compiler/cpp-dest-dir-initializer.h" #include "compiler/code-gen/code-gen-task.h" #include "compiler/code-gen/code-generator.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/files/cmake-lists-txt.h" +#include "compiler/code-gen/files/const-vars-init.h" #include "compiler/code-gen/files/function-header.h" #include "compiler/code-gen/files/function-source.h" -#include "compiler/code-gen/files/json-encoder-tags.h" -#include "compiler/code-gen/files/global_vars_memory_stats.h" +#include "compiler/code-gen/files/global-vars-memory-stats.h" +#include "compiler/code-gen/files/global-vars-reset.h" #include "compiler/code-gen/files/init-scripts.h" +#include "compiler/code-gen/files/json-encoder-tags.h" #include "compiler/code-gen/files/lib-header.h" -#include "compiler/code-gen/files/tl2cpp/tl2cpp.h" #include "compiler/code-gen/files/shape-keys.h" +#include "compiler/code-gen/files/tl2cpp/tl2cpp.h" #include "compiler/code-gen/files/tracing-autogen.h" #include "compiler/code-gen/files/type-tagger.h" -#include "compiler/code-gen/files/vars-cpp.h" -#include "compiler/code-gen/files/vars-reset.h" #include "compiler/code-gen/raw-data.h" #include "compiler/compiler-core.h" +#include "compiler/cpp-dest-dir-initializer.h" #include "compiler/data/class-data.h" #include "compiler/data/function-data.h" #include "compiler/data/generics-mixins.h" @@ -33,11 +34,6 @@ #include "compiler/pipes/collect-forkable-types.h" #include "compiler/type-hint.h" -size_t CodeGenF::calc_count_of_parts(size_t cnt_global_vars) { - return 1u + cnt_global_vars / G->settings().globals_split_count.get(); -} - - void CodeGenF::execute(FunctionPtr function, DataStream> &unused_os __attribute__ ((unused))) { if (function->does_need_codegen() || function->is_imported_from_static_lib()) { prepare_generate_function(function); @@ -60,6 +56,15 @@ void CodeGenF::on_finish(DataStream> &os) { const std::vector &all_classes = G->get_classes(); std::set all_json_encoders; + std::vector all_globals = G->get_global_vars(); + for (FunctionPtr f : all_functions) { + all_globals.insert(all_globals.end(), f->static_var_ids.begin(), f->static_var_ids.end()); + } + const GlobalsBatchedMem &all_globals_in_mem = GlobalsBatchedMem::prepare_mem_and_assign_offsets(all_globals); + + std::vector all_constants = G->get_constants_vars(); + const ConstantsBatchedMem &all_constants_in_mem = ConstantsBatchedMem::prepare_mem_and_assign_offsets(all_constants); + for (FunctionPtr f : all_functions) { code_gen_start_root_task(os, std::make_unique(f)); code_gen_start_root_task(os, std::make_unique(f)); @@ -90,36 +95,14 @@ void CodeGenF::on_finish(DataStream> &os) { } } - code_gen_start_root_task(os, std::make_unique(G->get_main_file())); - if (G->settings().enable_global_vars_memory_stats.get()) { - code_gen_start_root_task(os, std::make_unique(G->get_main_file())); + if (G->settings().enable_global_vars_memory_stats.get() && !G->is_output_mode_lib()) { + code_gen_start_root_task(os, std::make_unique(all_globals)); } code_gen_start_root_task(os, std::make_unique(G->get_main_file())); + code_gen_start_root_task(os, std::make_unique(all_constants_in_mem)); + code_gen_start_root_task(os, std::make_unique(all_globals_in_mem)); - std::vector vars = G->get_global_vars(); - for (FunctionPtr f : all_functions) { - vars.insert(vars.end(), f->static_var_ids.begin(), f->static_var_ids.end()); - } - size_t parts_cnt = calc_count_of_parts(vars.size()); - - std::vector> vars_batches(parts_cnt); - std::vector max_dep_levels(parts_cnt); - for (VarPtr var : vars) { - vars_batches[vk::std_hash(var->name) % parts_cnt].emplace_back(var); - } - for (size_t part_id = 0; part_id < parts_cnt; ++part_id) { - int max_dep_level{0}; - for (auto var : vars_batches[part_id]) { - if (var->is_constant() && max_dep_level < var->dependency_level) { - max_dep_level = var->dependency_level; - } - } - max_dep_levels[part_id] = max_dep_level; - code_gen_start_root_task(os, std::make_unique(std::move(vars_batches[part_id]), part_id)); - } - code_gen_start_root_task(os, std::make_unique(std::move(max_dep_levels), parts_cnt)); - - if (G->settings().is_static_lib_mode()) { + if (G->is_output_mode_lib()) { std::vector exported_functions; for (FunctionPtr f : all_functions) { if (f->kphp_lib_export) { @@ -133,7 +116,7 @@ void CodeGenF::on_finish(DataStream> &os) { } // TODO: should be done in lib mode also, but in some other way - if (!G->settings().is_static_lib_mode()) { + if (!G->is_output_mode_lib()) { code_gen_start_root_task(os, std::make_unique(vk::singleton::get().flush_forkable_types(), vk::singleton::get().flush_waitable_types())); code_gen_start_root_task(os, std::make_unique(TypeHintShape::get_all_registered_keys())); code_gen_start_root_task(os, std::make_unique(std::move(all_json_encoders))); @@ -146,7 +129,7 @@ void CodeGenF::on_finish(DataStream> &os) { code_gen_start_root_task(os, std::make_unique()); code_gen_start_root_task(os, std::make_unique()); - if (!G->settings().is_static_lib_mode()) { + if (!G->is_output_mode_lib()) { code_gen_start_root_task(os, std::make_unique()); } } @@ -185,9 +168,9 @@ void CodeGenF::prepare_generate_function(FunctionPtr func) { ? func->file_id->owner_lib->headers_dir() + func->header_name : func->subdir + "/" + func->header_name; - std::sort(func->static_var_ids.begin(), func->static_var_ids.end()); - std::sort(func->global_var_ids.begin(), func->global_var_ids.end()); - std::sort(func->local_var_ids.begin(), func->local_var_ids.end()); + std::sort(func->local_var_ids.begin(), func->local_var_ids.end(), [](VarPtr v1, VarPtr v2) { + return v1->name.compare(v2->name) < 0; + }); if (func->kphp_tracing) { TracingAutogen::register_function_marked_kphp_tracing(func); diff --git a/compiler/pipes/code-gen.h b/compiler/pipes/code-gen.h index 58aa425400..7b3f98fae8 100644 --- a/compiler/pipes/code-gen.h +++ b/compiler/pipes/code-gen.h @@ -19,7 +19,6 @@ class CodeGenF final : public SyncPipeF> &unused_os) final; diff --git a/compiler/pipes/collect-const-vars.cpp b/compiler/pipes/collect-const-vars.cpp index 001e734f0b..e0d774d625 100644 --- a/compiler/pipes/collect-const-vars.cpp +++ b/compiler/pipes/collect-const-vars.cpp @@ -7,6 +7,7 @@ #include "compiler/data/src-file.h" #include "compiler/vertex-util.h" #include "compiler/data/var-data.h" +#include "compiler/const-manipulations.h" #include "compiler/compiler-core.h" #include "compiler/name-gen.h" @@ -142,6 +143,34 @@ struct NameGenerator : public VertexVisitor { } return fallback(v); } + +private: + // checks that inlined as define' value constructor is suitable to be stored as constant var + static bool is_object_suitable_for_hashing(VertexPtr vertex) { + return vertex->type() == op_define_val && vertex.as()->value()->type() == op_func_call + && vertex.as()->value()->extra_type == op_ex_constructor_call && vertex->const_type == cnst_const_val; + } + + static bool is_array_suitable_for_hashing(VertexPtr vertex) { + return vertex->type() == op_array && CheckConst::is_const(vertex); + } + + static std::string gen_const_string_name(const std::string &str) { + return fmt_format("c_str${:x}", vk::std_hash(str)); + } + + static std::string gen_const_regexp_name(const std::string &str) { + return fmt_format("c_reg${:x}", vk::std_hash(str)); + } + + static std::string gen_const_object_name(const VertexAdaptor &def) { + kphp_assert_msg(def->value()->type() == op_func_call, "Internal error: expected op_define_val "); + return fmt_format("c_obj${:x}", ObjectHash::calc_hash(def)); + } + + static std::string gen_const_array_name(const VertexAdaptor &array) { + return fmt_format("c_arr${:x}", ArrayHash::calc_hash(array)); + } }; struct ProcessBeforeReplace : public VertexVisitor { @@ -203,7 +232,8 @@ void set_var_dep_level(VarPtr var_id) { VertexPtr CollectConstVarsPass::on_exit_vertex(VertexPtr root) { if (root->const_type == cnst_const_val) { composite_const_depth_ -= static_cast(IsComposite::visit(root)); - if (ShouldStoreOnBottomUp::visit(root)) { + if (ShouldStoreOnBottomUp::visit(root) + && !current_function->is_extern()) { // don't extract constants from extern func default arguments, they are in C++ runtime root = ProcessBeforeReplace::visit(root); root = create_const_variable(root, root->location); } @@ -242,7 +272,7 @@ VertexPtr CollectConstVarsPass::create_const_variable(VertexPtr root, Location l var->extra_type = op_ex_var_const; var->location = loc; - VarPtr var_id = G->get_global_var(name, VarData::var_const_t, VertexUtil::unwrap_inlined_define(root)); + VarPtr var_id = G->get_constant_var(name, VertexUtil::unwrap_inlined_define(root)); set_var_dep_level(var_id); if (composite_const_depth_ > 0) { diff --git a/compiler/pipes/convert-sprintf-calls.cpp b/compiler/pipes/convert-sprintf-calls.cpp index 6bad5fb214..06fe542ac9 100644 --- a/compiler/pipes/convert-sprintf-calls.cpp +++ b/compiler/pipes/convert-sprintf-calls.cpp @@ -140,7 +140,7 @@ VertexPtr ConvertSprintfCallsPass::convert_sprintf_call(VertexAdaptorlocation); vertex_parts.push_back(vertex); if (part.is_specifier()) { @@ -148,10 +148,10 @@ VertexPtr ConvertSprintfCallsPass::convert_sprintf_call(VertexAdaptor::create(vertex_parts); + return VertexAdaptor::create(vertex_parts).set_location(call->location); } -VertexPtr ConvertSprintfCallsPass::convert_format_part_to_vertex(const FormatPart &part, size_t arg_index, const FormatCallInfo &info) { +VertexPtr ConvertSprintfCallsPass::convert_format_part_to_vertex(const FormatPart &part, size_t arg_index, const FormatCallInfo &info, const Location &call_location) { if (part.is_specifier()) { VertexPtr element; @@ -180,7 +180,7 @@ VertexPtr ConvertSprintfCallsPass::convert_format_part_to_vertex(const FormatPar return VertexAdaptor::create(convert); } - VertexAdaptor vertex = VertexAdaptor::create(); + VertexAdaptor vertex = VertexAdaptor::create().set_location(call_location); vertex->set_string(part.value); return vertex; } diff --git a/compiler/pipes/convert-sprintf-calls.h b/compiler/pipes/convert-sprintf-calls.h index 55d711bf55..5f14308630 100644 --- a/compiler/pipes/convert-sprintf-calls.h +++ b/compiler/pipes/convert-sprintf-calls.h @@ -37,5 +37,5 @@ class ConvertSprintfCallsPass final : public FunctionPassBase { private: static VertexPtr convert_sprintf_call(VertexAdaptor call); - static VertexPtr convert_format_part_to_vertex(const FormatPart &part, size_t arg_index, const FormatCallInfo &info); + static VertexPtr convert_format_part_to_vertex(const FormatPart &part, size_t arg_index, const FormatCallInfo &info, const Location &call_location); }; diff --git a/compiler/pipes/final-check.cpp b/compiler/pipes/final-check.cpp index fe8ec17f97..7593c94b0b 100644 --- a/compiler/pipes/final-check.cpp +++ b/compiler/pipes/final-check.cpp @@ -14,7 +14,6 @@ #include "compiler/data/kphp-tracing-tags.h" #include "compiler/data/src-file.h" #include "compiler/data/var-data.h" -#include "compiler/data/vars-collector.h" #include "compiler/vertex-util.h" #include "compiler/type-hint.h" @@ -324,23 +323,12 @@ void check_register_shutdown_functions(VertexAdaptor call) { vk::join(throws, ", "), callback->func_id->get_throws_call_chain())); } -void mark_global_vars_for_memory_stats() { - if (!G->settings().enable_global_vars_memory_stats.get()) { - return; - } - - static std::atomic vars_marked{false}; - if (vars_marked.exchange(true)) { - return; - } - +void mark_global_vars_for_memory_stats(const std::vector &vars_list) { std::unordered_set classes_inside; - VarsCollector vars_collector{0, [&classes_inside](VarPtr variable) { - tinf::get_type(variable)->get_all_class_types_inside(classes_inside); - return false; - }}; - vars_collector.collect_global_and_static_vars_from(G->get_main_file()->main_function); - for (auto klass: classes_inside) { + for (VarPtr var : vars_list) { + tinf::get_type(var)->get_all_class_types_inside(classes_inside); + } + for (ClassPtr klass: classes_inside) { klass->deeply_require_instance_memory_estimate_visitor(); } } @@ -559,7 +547,15 @@ void check_php2c_conv(VertexAdaptor conv) { } // namespace void FinalCheckPass::on_start() { - mark_global_vars_for_memory_stats(); + if (G->settings().enable_global_vars_memory_stats.get()) { + static std::atomic globals_marked{false}; + if (!globals_marked.exchange(true)) { + mark_global_vars_for_memory_stats(G->get_global_vars()); + } + if (!current_function->static_var_ids.empty()) { + mark_global_vars_for_memory_stats(current_function->static_var_ids); + } + } if (current_function->type == FunctionData::func_class_holder) { check_class_immutableness(current_function->class_id); diff --git a/compiler/pipes/gen-tree-postprocess.cpp b/compiler/pipes/gen-tree-postprocess.cpp index ddd9b2a94b..fd1f1c81b4 100644 --- a/compiler/pipes/gen-tree-postprocess.cpp +++ b/compiler/pipes/gen-tree-postprocess.cpp @@ -90,7 +90,7 @@ VertexAdaptor make_require_once_call(SrcFilePtr lib_main_file, Verte } VertexPtr process_require_lib(VertexAdaptor require_lib_call) { - kphp_error_act (!G->settings().is_static_lib_mode(), "require_lib is forbidden to use for compiling libs", return require_lib_call); + kphp_error_act (!G->is_output_mode_lib(), "require_lib is forbidden to use for compiling libs", return require_lib_call); VertexRange args = require_lib_call->args(); kphp_error_act (args.size() == 1, fmt_format("require_lib expected 1 arguments, got {}", args.size()), return require_lib_call); auto lib_name_node = args[0]; @@ -230,9 +230,9 @@ VertexPtr GenTreePostprocessPass::on_enter_vertex(VertexPtr root) { } VertexPtr GenTreePostprocessPass::on_exit_vertex(VertexPtr root) { - if (root->type() == op_var) { - if (VertexUtil::is_superglobal(root->get_string())) { - root->extra_type = op_ex_var_superglobal; + if (auto as_var = root.try_as()) { + if (as_var->str_val[0] == '_' && VarData::does_name_eq_any_language_superglobal(as_var->str_val)) { + as_var->extra_type = op_ex_var_superglobal; } } diff --git a/compiler/pipes/generate-virtual-methods.cpp b/compiler/pipes/generate-virtual-methods.cpp index 32053a863d..681b564473 100644 --- a/compiler/pipes/generate-virtual-methods.cpp +++ b/compiler/pipes/generate-virtual-methods.cpp @@ -423,7 +423,7 @@ void generate_body_of_virtual_method(FunctionPtr virtual_function) { } } if (!cases.empty()) { - auto case_default_warn = generate_critical_error_call(fmt_format("call method({}) on null object", virtual_function->as_human_readable())); + auto case_default_warn = generate_critical_error_call(fmt_format("call method({}) on null object", virtual_function->as_human_readable(false))); cases.emplace_back(VertexAdaptor::create(VertexAdaptor::create(case_default_warn))); } diff --git a/compiler/pipes/optimization.cpp b/compiler/pipes/optimization.cpp index c7ef4756d5..5fb9503ae4 100644 --- a/compiler/pipes/optimization.cpp +++ b/compiler/pipes/optimization.cpp @@ -45,7 +45,7 @@ VarPtr cast_const_array_type(VertexPtr &type_acceptor, const TypeData *required_ ss << type_acceptor->get_string() << "$" << std::hex << vk::std_hash(type_out(required_type)); std::string name = ss.str(); bool is_new = true; - VarPtr var_id = G->get_global_var(name, VarData::var_const_t, type_acceptor, &is_new); + VarPtr var_id = G->get_constant_var(name, type_acceptor, &is_new); var_id->tinf_node.copy_type_from(required_type); // not inside if(is_new) to avoid race conditions when one thread creates and another uses faster if (is_new) { var_id->dependency_level = type_acceptor.as()->var_id->dependency_level + 1; diff --git a/compiler/pipes/register-variables.cpp b/compiler/pipes/register-variables.cpp index cd3d153ce6..30b8f80290 100644 --- a/compiler/pipes/register-variables.cpp +++ b/compiler/pipes/register-variables.cpp @@ -12,7 +12,7 @@ #include "compiler/utils/string-utils.h" VarPtr RegisterVariablesPass::create_global_var(const std::string &name) { - VarPtr var = G->get_global_var(name, VarData::var_global_t, VertexPtr()); + VarPtr var = G->get_global_var(name, VertexPtr()); auto it = registred_vars.insert(make_pair(name, var)); if (it.second == false) { VarPtr old_var = it.first->second; diff --git a/compiler/pipes/remove-empty-function-calls.cpp b/compiler/pipes/remove-empty-function-calls.cpp index cdefe84b0d..6595ed5d0f 100644 --- a/compiler/pipes/remove-empty-function-calls.cpp +++ b/compiler/pipes/remove-empty-function-calls.cpp @@ -46,7 +46,7 @@ VertexPtr RemoveEmptyFunctionCallsPass::on_exit_vertex(VertexPtr v) { // get rid of $called - global variables for empty source files; // namely, detect 'v$src_fooxxx$called = true' assign in such files and remove it, // this allows to avoid further call of register_var() with such global variable - if (!G->settings().is_static_lib_mode() && current_function->is_main_function() && current_function->body_seq == FunctionData::body_value::empty) { + if (!G->is_output_mode_lib() && current_function->is_main_function() && current_function->body_seq == FunctionData::body_value::empty) { auto set = v.as(); auto lhs = set->lhs(); auto rhs = set->rhs(); diff --git a/compiler/threading/hash-table.h b/compiler/threading/hash-table.h index 4fa9189d4b..010be0efe9 100644 --- a/compiler/threading/hash-table.h +++ b/compiler/threading/hash-table.h @@ -64,6 +64,7 @@ class TSHashTable { std::vector get_all() { std::vector res; + res.reserve(used_size); for (int i = 0; i < N; i++) { if (nodes[i].hash != 0) { res.push_back(nodes[i].data); diff --git a/compiler/vertex-util.cpp b/compiler/vertex-util.cpp index 7f6d1606c8..56a01608da 100644 --- a/compiler/vertex-util.cpp +++ b/compiler/vertex-util.cpp @@ -116,19 +116,6 @@ void VertexUtil::func_force_return(VertexAdaptor func, VertexPtr va func->cmd_ref() = VertexAdaptor::create(next); } -bool VertexUtil::is_superglobal(const std::string &s) { - static std::set names = { - "_SERVER", - "_GET", - "_POST", - "_FILES", - "_COOKIE", - "_REQUEST", - "_ENV" - }; - return vk::contains(names, s); -} - bool VertexUtil::is_positive_constexpr_int(VertexPtr v) { auto actual_value = get_actual_value(v).try_as(); return actual_value && parse_int_from_string(actual_value) >= 0; diff --git a/compiler/vertex-util.h b/compiler/vertex-util.h index 24bedc2a5f..c118e7a3a9 100644 --- a/compiler/vertex-util.h +++ b/compiler/vertex-util.h @@ -36,7 +36,6 @@ class VertexUtil { static void func_force_return(VertexAdaptor func, VertexPtr val = {}); - static bool is_superglobal(const std::string &s); static bool is_positive_constexpr_int(VertexPtr v); static bool is_const_int(VertexPtr root); }; diff --git a/docs/kphp-language/kphp-vs-php/compiler-cmd-options.md b/docs/kphp-language/kphp-vs-php/compiler-cmd-options.md index 1fde19e6df..b67550a870 100644 --- a/docs/kphp-language/kphp-vs-php/compiler-cmd-options.md +++ b/docs/kphp-language/kphp-vs-php/compiler-cmd-options.md @@ -75,10 +75,6 @@ Threads number for PHP → C++ codegeneration, default **CPU cores * 2**. Processes number to C++ parallel compilation/linkage, default **CPU cores**. - - -All global variables (const arrays also) are split into chunks of this size, default **1024**. If you have a few but very heavy global vars, lowering this number can decrease compilation time. - A *.tl* file with [TL schema](../../kphp-client/tl-schema-and-rpc/tl-schema-basics.md), default empty. diff --git a/runtime/array_functions.cpp b/runtime/array_functions.cpp index 1b526e1f42..1589dcb5c6 100644 --- a/runtime/array_functions.cpp +++ b/runtime/array_functions.cpp @@ -257,3 +257,6 @@ string implode_string_vector(const string &s, const array &a) { } return result.finish_append(); } + +static_assert(sizeof(array) == SIZEOF_ARRAY_ANY, "sizeof(array) at runtime doesn't match compile-time"); + diff --git a/runtime/interface.cpp b/runtime/interface.cpp index 16c327d864..be036860e7 100644 --- a/runtime/interface.cpp +++ b/runtime/interface.cpp @@ -595,7 +595,7 @@ void f$fastcgi_finish_request(int64_t exit_code) { write_safe(1, oub[ob_total_buffer].buffer(), oub[ob_total_buffer].size(), {}); //TODO move to finish_script - free_runtime_environment(); + free_runtime_environment(PhpScriptMutableGlobals::current().get_superglobals()); break; } @@ -866,8 +866,6 @@ bool f$get_magic_quotes_gpc() { return false; } -string v$d$PHP_SAPI __attribute__ ((weak)); - static string php_sapi_name() { switch (query_type) { @@ -890,21 +888,10 @@ static string php_sapi_name() { } string f$php_sapi_name() { - return v$d$PHP_SAPI; + return PhpScriptMutableGlobals::current().get_superglobals().v$d$PHP_SAPI; } -mixed v$_SERVER __attribute__ ((weak)); -mixed v$_GET __attribute__ ((weak)); -mixed v$_POST __attribute__ ((weak)); -mixed v$_FILES __attribute__ ((weak)); -mixed v$_COOKIE __attribute__ ((weak)); -mixed v$_REQUEST __attribute__ ((weak)); -mixed v$_ENV __attribute__ ((weak)); - -mixed v$argc __attribute__ ((weak)); -mixed v$argv __attribute__ ((weak)); - static std::aligned_storage_t), alignof(array)> uploaded_files_storage; static array *uploaded_files = reinterpret_cast *> (&uploaded_files_storage); static long long uploaded_files_last_query_num = -1; @@ -1176,7 +1163,7 @@ class post_reader { } }; -static int parse_multipart_one(post_reader &data, int i) { +static int parse_multipart_one(post_reader &data, int i, mixed &v$_POST, mixed &v$_FILES) { string content_type("text/plain", 10); string name; string filename; @@ -1352,7 +1339,7 @@ static int parse_multipart_one(post_reader &data, int i) { return i; } -static bool parse_multipart(const char *post, int post_len, const string &boundary) { +static bool parse_multipart(const char *post, int post_len, const string &boundary, mixed &v$_POST, mixed &v$_FILES) { static const int MAX_BOUNDARY_LENGTH = 70; if (boundary.empty() || (int)boundary.size() > MAX_BOUNDARY_LENGTH) { @@ -1364,7 +1351,7 @@ static bool parse_multipart(const char *post, int post_len, const string &bounda for (int i = 0; i < post_len; i++) { // fprintf (stderr, "!!!! %d\n", i); - i = parse_multipart_one(data, i); + i = parse_multipart_one(data, i, v$_POST, v$_FILES); // fprintf (stderr, "???? %d\n", i); while (!data.is_boundary(i)) { @@ -1461,16 +1448,16 @@ void arg_add(const char *value) { arg_vars->push_back(string(value)); } -static void reset_superglobals() { +static void reset_superglobals(PhpScriptBuiltInSuperGlobals &superglobals) { dl::enter_critical_section(); - hard_reset_var(v$_SERVER, array()); - hard_reset_var(v$_GET, array()); - hard_reset_var(v$_POST, array()); - hard_reset_var(v$_FILES, array()); - hard_reset_var(v$_COOKIE, array()); - hard_reset_var(v$_REQUEST, array()); - hard_reset_var(v$_ENV, array()); + hard_reset_var(superglobals.v$_SERVER, array()); + hard_reset_var(superglobals.v$_GET, array()); + hard_reset_var(superglobals.v$_POST, array()); + hard_reset_var(superglobals.v$_ENV, array()); + hard_reset_var(superglobals.v$_FILES, array()); + hard_reset_var(superglobals.v$_COOKIE, array()); + hard_reset_var(superglobals.v$_REQUEST, array()); dl::leave_critical_section(); } @@ -1478,7 +1465,7 @@ static void reset_superglobals() { // RFC link: https://tools.ietf.org/html/rfc2617#section-2 // Header example: // Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ== -static void parse_http_authorization_header(const string &header_value) { +static void parse_http_authorization_header(const string &header_value, mixed &v$_SERVER) { array header_parts = explode(' ', header_value); if (header_parts.count() != 2) { return; @@ -1501,7 +1488,7 @@ static void parse_http_authorization_header(const string &header_value) { v$_SERVER.set_value(string("AUTH_TYPE"), auth_scheme); } -static void save_rpc_query_headers(const tl_query_header_t &header) { +static void save_rpc_query_headers(const tl_query_header_t &header, mixed &v$_SERVER) { namespace flag = vk::tl::common::rpc_invoke_req_extra_flags; if (header.actor_id) { @@ -1546,37 +1533,37 @@ static void save_rpc_query_headers(const tl_query_header_t &header) { } } -static void init_superglobals_impl(const http_query_data &http_data, const rpc_query_data &rpc_data, const job_query_data &job_data) { +static void init_superglobals_impl(const http_query_data &http_data, const rpc_query_data &rpc_data, const job_query_data &job_data, PhpScriptBuiltInSuperGlobals &superglobals) { rpc_parse(rpc_data.data.data(), rpc_data.data.size()); - reset_superglobals(); + reset_superglobals(superglobals); if (query_type == QUERY_TYPE_JOB) { - v$_SERVER.set_value(string("JOB_ID"), job_data.job_request->job_id); + superglobals.v$_SERVER.set_value(string("JOB_ID"), job_data.job_request->job_id); init_job_server_interface_lib(job_data); } string uri_str; if (http_data.uri_len) { uri_str.assign(http_data.uri, http_data.uri_len); - v$_SERVER.set_value(string("PHP_SELF"), uri_str); - v$_SERVER.set_value(string("SCRIPT_URL"), uri_str); - v$_SERVER.set_value(string("SCRIPT_NAME"), uri_str); + superglobals.v$_SERVER.set_value(string("PHP_SELF"), uri_str); + superglobals.v$_SERVER.set_value(string("SCRIPT_URL"), uri_str); + superglobals.v$_SERVER.set_value(string("SCRIPT_NAME"), uri_str); } string get_str; if (http_data.get_len) { get_str.assign(http_data.get, http_data.get_len); - f$parse_str(get_str, v$_GET); + f$parse_str(get_str, superglobals.v$_GET); - v$_SERVER.set_value(string("QUERY_STRING"), get_str); + superglobals.v$_SERVER.set_value(string("QUERY_STRING"), get_str); } if (http_data.uri) { if (http_data.get_len) { - v$_SERVER.set_value(string("REQUEST_URI"), (static_SB.clean() << uri_str << '?' << get_str).str()); + superglobals.v$_SERVER.set_value(string("REQUEST_URI"), (static_SB.clean() << uri_str << '?' << get_str).str()); } else { - v$_SERVER.set_value(string("REQUEST_URI"), uri_str); + superglobals.v$_SERVER.set_value(string("REQUEST_URI"), uri_str); } } @@ -1624,13 +1611,13 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q for (int t = 0; t < (int)cookie.count(); t++) { array cur_cookie = explode('=', f$trim(cookie[t]), 2); if ((int)cur_cookie.count() == 2) { - parse_str_set_value(v$_COOKIE, cur_cookie[0], f$urldecode(cur_cookie[1])); + parse_str_set_value(superglobals.v$_COOKIE, cur_cookie[0], f$urldecode(cur_cookie[1])); } } } else if (!strcmp(header_name.c_str(), "host")) { - v$_SERVER.set_value(string("SERVER_NAME"), header_value); + superglobals.v$_SERVER.set_value(string("SERVER_NAME"), header_value); } else if (!strcmp(header_name.c_str(), "authorization")) { - parse_http_authorization_header(header_value); + parse_http_authorization_header(header_value, superglobals.v$_SERVER); } if (!strcmp(header_name.c_str(), "content-type")) { @@ -1659,7 +1646,7 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q key[2] = 'T'; key[3] = 'P'; key[4] = '_'; - v$_SERVER.set_value(key, header_value); + superglobals.v$_SERVER.set_value(key, header_value); } else { // fprintf (stderr, "%s : %s\n", header_name.c_str(), header_value.c_str()); } @@ -1670,12 +1657,12 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q string HTTP_X_REAL_SCHEME("HTTP_X_REAL_SCHEME", 18); string HTTP_X_REAL_HOST("HTTP_X_REAL_HOST", 16); string HTTP_X_REAL_REQUEST("HTTP_X_REAL_REQUEST", 19); - if (v$_SERVER.isset(HTTP_X_REAL_SCHEME) && v$_SERVER.isset(HTTP_X_REAL_HOST) && v$_SERVER.isset(HTTP_X_REAL_REQUEST)) { - string script_uri(v$_SERVER.get_value(HTTP_X_REAL_SCHEME).to_string()); + if (superglobals.v$_SERVER.isset(HTTP_X_REAL_SCHEME) && superglobals.v$_SERVER.isset(HTTP_X_REAL_HOST) && superglobals.v$_SERVER.isset(HTTP_X_REAL_REQUEST)) { + string script_uri(superglobals.v$_SERVER.get_value(HTTP_X_REAL_SCHEME).to_string()); script_uri.append("://", 3); - script_uri.append(v$_SERVER.get_value(HTTP_X_REAL_HOST).to_string()); - script_uri.append(v$_SERVER.get_value(HTTP_X_REAL_REQUEST).to_string()); - v$_SERVER.set_value(string("SCRIPT_URI"), script_uri); + script_uri.append(superglobals.v$_SERVER.get_value(HTTP_X_REAL_HOST).to_string()); + script_uri.append(superglobals.v$_SERVER.get_value(HTTP_X_REAL_REQUEST).to_string()); + superglobals.v$_SERVER.set_value(string("SCRIPT_URI"), script_uri); } if (http_data.post_len > 0) { @@ -1687,7 +1674,7 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q raw_post_data.assign(http_data.post, http_data.post_len); dl::leave_critical_section(); - f$parse_str(raw_post_data, v$_POST); + f$parse_str(raw_post_data, superglobals.v$_POST); } } else if (strstr(content_type_lower.c_str(), "multipart/form-data")) { const char *p = strstr(content_type_lower.c_str(), "boundary"); @@ -1703,7 +1690,7 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q end_p--; } // fprintf (stderr, "!%s!\n", p); - is_parsed |= parse_multipart(http_data.post, http_data.post_len, string(p, static_cast(end_p - p))); + is_parsed |= parse_multipart(http_data.post, http_data.post_len, string(p, static_cast(end_p - p)), superglobals.v$_POST, superglobals.v$_FILES); } } } else { @@ -1723,51 +1710,51 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q } } - v$_SERVER.set_value(string("CONTENT_TYPE"), content_type); + superglobals.v$_SERVER.set_value(string("CONTENT_TYPE"), content_type); } double cur_time = microtime(); - v$_SERVER.set_value(string("GATEWAY_INTERFACE"), string("CGI/1.1")); + superglobals.v$_SERVER.set_value(string("GATEWAY_INTERFACE"), string("CGI/1.1")); if (http_data.ip) { - v$_SERVER.set_value(string("REMOTE_ADDR"), f$long2ip(static_cast(http_data.ip))); + superglobals.v$_SERVER.set_value(string("REMOTE_ADDR"), f$long2ip(static_cast(http_data.ip))); } if (http_data.port) { - v$_SERVER.set_value(string("REMOTE_PORT"), static_cast(http_data.port)); + superglobals.v$_SERVER.set_value(string("REMOTE_PORT"), static_cast(http_data.port)); } if (rpc_data.header.qid) { - v$_SERVER.set_value(string("RPC_REQUEST_ID"), f$strval(static_cast(rpc_data.header.qid))); - save_rpc_query_headers(rpc_data.header); - v$_SERVER.set_value(string("RPC_REMOTE_IP"), static_cast(rpc_data.remote_pid.ip)); - v$_SERVER.set_value(string("RPC_REMOTE_PORT"), static_cast(rpc_data.remote_pid.port)); - v$_SERVER.set_value(string("RPC_REMOTE_PID"), static_cast(rpc_data.remote_pid.pid)); - v$_SERVER.set_value(string("RPC_REMOTE_UTIME"), rpc_data.remote_pid.utime); + superglobals.v$_SERVER.set_value(string("RPC_REQUEST_ID"), f$strval(static_cast(rpc_data.header.qid))); + save_rpc_query_headers(rpc_data.header, superglobals.v$_SERVER); + superglobals.v$_SERVER.set_value(string("RPC_REMOTE_IP"), static_cast(rpc_data.remote_pid.ip)); + superglobals.v$_SERVER.set_value(string("RPC_REMOTE_PORT"), static_cast(rpc_data.remote_pid.port)); + superglobals.v$_SERVER.set_value(string("RPC_REMOTE_PID"), static_cast(rpc_data.remote_pid.pid)); + superglobals.v$_SERVER.set_value(string("RPC_REMOTE_UTIME"), rpc_data.remote_pid.utime); } is_head_query = false; if (http_data.request_method_len) { - v$_SERVER.set_value(string("REQUEST_METHOD"), string(http_data.request_method, http_data.request_method_len)); + superglobals.v$_SERVER.set_value(string("REQUEST_METHOD"), string(http_data.request_method, http_data.request_method_len)); if (http_data.request_method_len == 4 && !strncmp(http_data.request_method, "HEAD", http_data.request_method_len)) { is_head_query = true; } } - v$_SERVER.set_value(string("REQUEST_TIME"), int(cur_time)); - v$_SERVER.set_value(string("REQUEST_TIME_FLOAT"), cur_time); - v$_SERVER.set_value(string("SERVER_PORT"), string("80")); - v$_SERVER.set_value(string("SERVER_PROTOCOL"), string("HTTP/1.1")); - v$_SERVER.set_value(string("SERVER_SIGNATURE"), (static_SB.clean() << "Apache/2.2.9 (Debian) PHP/5.2.6-1<second; +} diff --git a/runtime/php-script-globals.h b/runtime/php-script-globals.h new file mode 100644 index 0000000000..028a165c56 --- /dev/null +++ b/runtime/php-script-globals.h @@ -0,0 +1,49 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime/kphp_core.h" + +struct PhpScriptBuiltInSuperGlobals { + // variables below are PHP language superglobals + mixed v$_SERVER; + mixed v$_GET; + mixed v$_POST; + mixed v$_ENV; + mixed v$_FILES; + mixed v$_COOKIE; + mixed v$_REQUEST; + + // variables below are not superglobals of the PHP language, but since they are set by runtime, + // the compiler is also aware about them + mixed v$argc; + mixed v$argv; + string v$d$PHP_SAPI; // define('PHP_SAPI') +}; + +// storage of linear memory used for mutable globals in each script +// on worker start, once_alloc_linear_mem() is called from codegen +// it initializes g_linear_mem, and every mutable global access is codegenerated +// as smth line `(*reinterpret_cast(&php_globals.mem()+offset))` +class PhpScriptMutableGlobals { + char *g_linear_mem{nullptr}; + std::unordered_map libs_linear_mem; + PhpScriptBuiltInSuperGlobals superglobals; + +public: + static PhpScriptMutableGlobals ¤t(); + ~PhpScriptMutableGlobals(); + + void once_alloc_linear_mem(unsigned int n_bytes); + void once_alloc_linear_mem(const char *lib_name, unsigned int n_bytes); + + char *mem() const { return g_linear_mem; } + char *mem_for_lib(const char *lib_name) const; + + PhpScriptBuiltInSuperGlobals &get_superglobals() { return superglobals; } + const PhpScriptBuiltInSuperGlobals &get_superglobals() const { return superglobals; } +}; diff --git a/runtime/regexp.cpp b/runtime/regexp.cpp index 21ba455b56..16669bf205 100644 --- a/runtime/regexp.cpp +++ b/runtime/regexp.cpp @@ -33,6 +33,7 @@ static re2::StringPiece RE2_submatch[MAX_SUBPATTERNS]; int32_t regexp::submatch[3 * MAX_SUBPATTERNS]; pcre_extra regexp::extra; +static_assert(sizeof(regexp) == SIZEOF_REGEXP, "sizeof(regexp) at runtime doesn't match compile-time"); regexp::regexp(const string ®exp_string) { init(regexp_string); diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index edd072a119..62215b7066 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -96,6 +96,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ math_functions.cpp mbstring.cpp memcache.cpp + memory_usage.cpp migration_php8.cpp misc.cpp mixed.cpp @@ -105,6 +106,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ oom_handler.cpp openssl.cpp php_assert.cpp + php-script-globals.cpp profiler.cpp regexp.cpp resumable.cpp diff --git a/runtime/string.cpp b/runtime/string.cpp index 13b570e14f..c405722055 100644 --- a/runtime/string.cpp +++ b/runtime/string.cpp @@ -8,3 +8,5 @@ string::~string() noexcept { destroy(); } + +static_assert(sizeof(string) == SIZEOF_STRING, "sizeof(string) at runtime doesn't match compile-time"); diff --git a/server/php-engine.cpp b/server/php-engine.cpp index 06f8273e90..3c2a1da794 100644 --- a/server/php-engine.cpp +++ b/server/php-engine.cpp @@ -1669,14 +1669,13 @@ void init_all() { StatsHouseManager::get().set_common_tags(); global_init_runtime_libs(); - global_init_php_scripts(); + init_php_scripts_once_in_master(); global_init_script_allocator(); init_handlers(); init_drivers(); - init_php_scripts(); vk::singleton::get().set_idle_worker_status(); worker_id = (int)lrand48(); diff --git a/server/php-init-scripts.cpp b/server/php-init-scripts.cpp index ed9e0f5af4..a275743cce 100644 --- a/server/php-init-scripts.cpp +++ b/server/php-init-scripts.cpp @@ -10,6 +10,6 @@ script_t *get_script() { return main_script; } -void set_script(void (*run)(), void (*clear)()) { +void set_script(void (*run)(), void (*clear)(PhpScriptMutableGlobals &php_globals)) { main_script = new script_t{run, clear}; } diff --git a/server/php-init-scripts.h b/server/php-init-scripts.h index 0e7b51f6b5..2c2193afde 100644 --- a/server/php-init-scripts.h +++ b/server/php-init-scripts.h @@ -6,19 +6,21 @@ #include +class PhpScriptMutableGlobals; + struct script_t { void (*run)(); // this is entrypoint to generated code - void (*clear)(); + void (*clear)(PhpScriptMutableGlobals &php_globals); }; -/// It binds generated code and runtime. Definition is generated by compiler. This function is called at global initialization @see init_all(). -void init_php_scripts() noexcept; -/// It initializes const variables. Definition is generated by compiler. This function is called at global initialization @see init_all(). -void global_init_php_scripts() noexcept; +/// Initializes const variables represented as globals C++ symbols. Definition is generated by compiler. +void init_php_scripts_once_in_master() noexcept; +/// Initializes mutable globals in a single linear memory piece. +void init_php_scripts_in_each_worker(PhpScriptMutableGlobals &php_globals) noexcept; script_t *get_script(); /// It's called from init_php_scripts() and set the entrypoint to generated code -void set_script(void (*run)(), void (*clear)()); +void set_script(void (*run)(), void (*clear)(PhpScriptMutableGlobals &php_globals)); struct script_result { const char *headers; diff --git a/server/php-runner.cpp b/server/php-runner.cpp index 45ce1805c1..24276fd132 100644 --- a/server/php-runner.cpp +++ b/server/php-runner.cpp @@ -359,8 +359,8 @@ void PhpScript::finish() noexcept { void PhpScript::clear() noexcept { assert_state(run_state_t::uncleared); - run_main->clear(); - free_runtime_environment(); + run_main->clear(PhpScriptMutableGlobals::current()); + free_runtime_environment(PhpScriptMutableGlobals::current().get_superglobals()); state = run_state_t::empty; if (use_madvise_dontneed) { if (dl::get_script_memory_stats().real_memory_used > memory_used_to_recreate_script) { @@ -413,7 +413,7 @@ void PhpScript::run() noexcept { in_script_context = true; auto oom_handling_memory_size = static_cast(std::ceil(mem_size * oom_handling_memory_ratio)); auto script_memory_size = mem_size - oom_handling_memory_size; - init_runtime_environment(*data, run_mem, script_memory_size, oom_handling_memory_size); + init_runtime_environment(*data, PhpScriptMutableGlobals::current().get_superglobals(), run_mem, script_memory_size, oom_handling_memory_size); dl::leave_critical_section(); php_assert (dl::in_critical_section == 0); // To ensure that no critical section is left at the end of the initialization check_net_context_errors(); diff --git a/tests/cpp/runtime/_runtime-tests-env.cpp b/tests/cpp/runtime/_runtime-tests-env.cpp index 15228c518b..2bfbc6d37c 100644 --- a/tests/cpp/runtime/_runtime-tests-env.cpp +++ b/tests/cpp/runtime/_runtime-tests-env.cpp @@ -30,7 +30,7 @@ class RuntimeTestsEnvironment final : public testing::Environment { global_init_runtime_libs(); global_init_script_allocator(); - init_runtime_environment(null_query_data{}, script_memory, script_memory_size); + init_runtime_environment(null_query_data{}, PhpScriptMutableGlobals::current().get_superglobals(), script_memory, script_memory_size); php_disable_warnings = true; php_warning_level = 0; } @@ -38,7 +38,7 @@ class RuntimeTestsEnvironment final : public testing::Environment { void TearDown() final { reset_global_vars(); - free_runtime_environment(); + free_runtime_environment(PhpScriptMutableGlobals::current().get_superglobals()); testing::Environment::TearDown(); } @@ -66,10 +66,11 @@ template<> int Storage::tagger>>::get_ template<> int Storage::tagger>::get_tag() noexcept { return 0; } template<> Storage::loader::loader_fun Storage::loader::get_function(int) noexcept { return nullptr; } -void init_php_scripts() noexcept { +void init_php_scripts_once_in_master() noexcept { assert(0 && "this code shouldn't be executed and only for linkage test"); } -void global_init_php_scripts() noexcept { +void init_php_scripts_in_each_worker(PhpScriptMutableGlobals &php_globals) noexcept { + static_cast(php_globals); assert(0 && "this code shouldn't be executed and only for linkage test"); } const char *get_php_scripts_version() noexcept { diff --git a/tests/phpt/constants/010_arrow_access.php b/tests/phpt/constants/010_arrow_access.php index f2c217e8d6..b4d2adfc30 100644 --- a/tests/phpt/constants/010_arrow_access.php +++ b/tests/phpt/constants/010_arrow_access.php @@ -3,10 +3,9 @@ require_once 'kphp_tester_include.php'; class B { - static int $static_int = 0; public int $value; public function __construct(int $x) { - B::$static_int += 1; + // B::$static_int += 1; // accessing globals in constructor in const classes crashes const_init; todo how to detect it in the future? $this->value = $x; } public function getValue() { diff --git a/tests/phpt/dl/1043_some_globals.php b/tests/phpt/dl/1043_some_globals.php new file mode 100644 index 0000000000..731b3629d1 --- /dev/null +++ b/tests/phpt/dl/1043_some_globals.php @@ -0,0 +1,291 @@ +@ok + [], + 'f' => [], + ]; +} + +$targets = getEmptyArrayOfArraysOfClass(); +echo 'count empty arr ', count($targets), "\n"; + +define('NOW', time()); +if (0) { + echo NOW, "\n"; +} + +// const_var +$s_concat = 'asdf' . '3'; +echo $s_concat, "\n"; +// const_var +$a_tuples = [tuple(1, true)]; +echo count($a_tuples); +// const_var float +$float_sin_30 = sin(30); +echo $float_sin_30, "\n"; +// const_var optional +$float_optional_min = min(1.0, null); +echo $float_optional_min, "\n"; +// const_var bool +$g_assigned_const_bool = min(true, false); +var_dump($g_assigned_const_bool); + +function acceptsVariadict(...$args) { + var_dump($args); +} +acceptsVariadict(''. 'str converted to array(1) into variadic'); + +/** @param mixed[] $arr */ +function acceptArrayMixed($arr) { + static $inAcceptArrayMixed = [0]; + + echo count($arr); + preg_match('/asdf+/', $arr[0]); +} + +acceptArrayMixed([1,2,3]); +acceptArrayMixed([4, 5, 6]); + +/** @var tuple(int, tuple(string|false, int|null)) */ +$g_tuple = tuple(1, tuple('str', 10)); +echo $g_tuple[1][0], "\n"; +$g_tuple = tuple(1, tuple(false, null)); +var_dump($g_tuple[1][1]); echo "\n"; + +/** @var tuple(bool) */ +$g_tuple_2 = tuple(true); +var_dump($g_tuple_2[0]); +/** @var tuple(bool,bool) */ +$g_tuple_3 = tuple(true,true); +var_dump($g_tuple_3[1]); +/** @var tuple(bool,int,bool) */ +$g_tuple_4 = tuple(true,0,true); +var_dump($g_tuple_4[2]); +/** @var tuple(bool,bool,bool,bool,bool,bool,bool,bool,bool) */ +$g_tuple_5 = tuple(true,true,true,true,true,true,true,false,true); +var_dump($g_tuple_5[7]); +var_dump($g_tuple_5[8]); + +/** @var shape(x:?int, y:SurveyTarget) */ +$g_shape = shape(['y' => new SurveyTarget]); +echo get_class($g_shape['y']), "\n"; + +function getInt(): int { return 5; } + +/** @var future */ +$g_future = fork(getInt()); +$g_future_result = wait($g_future); + +/** @var ?future_queue */ +$g_future_queue = wait_queue_create([$g_future]); + +function defArgConstants($s1 = 'str', $ar = [1,2,3]) { + $a = [[[[1]]]][0]; + echo $s1, count($ar), "\n"; +} +defArgConstants(); +defArgConstants('str2', [1,2]); + +class WithGlobals { + static public $unused_and_untyped = null; + static public $used_and_untyped = null; + static public int $c1_int = 0; + static public string $c1_string = 'asdf'; + /** @var ?SurveyTarget */ + static public $inst1 = null; + /** @var ?tuple(int, bool, ?SurveyTarget[]) */ + static public $tup1 = null; + /** @var ?tuple(bool) */ + static public $tup2 = null; + /** @var ?tuple(bool,bool,bool,?int,bool,bool,bool,bool) */ + static public $tup3 = null; + /** @var ?tuple(bool,bool,bool,bool,bool,bool,bool,bool,bool) */ + static public $tup4 = null; + /** @var ?shape(single: tuple(bool, mixed, bool)) */ + static public $sh1 = null; + /** @var Exception */ + static public $ex1 = null; + /** @var ?LogicException */ + static public $ex2 = null; + + static function use() { + self::$c1_string .= 'a' . 'b'; + self::$c1_int += 10; + + self::$tup1 = tuple(1, true, null); + self::$tup1 = tuple(1, true, [new SurveyTarget]); + self::$sh1 = shape([ + 'single' => tuple(true, [1, 'str'], false), + ]); + self::$tup3 = tuple(true,true,true,null,true,true,false,true); + self::$tup4 = tuple(true,true,true,true,true,true,true,false,true); + self::$ex1 = new Exception; + self::$ex2 = new LogicException; + + echo self::$used_and_untyped, "\n"; + echo self::$c1_int, "\n"; + echo self::$c1_string, "\n"; + echo self::$sh1['single'][1][1], "\n"; + + var_dump(self::$tup2 === null); + self::$tup2 = tuple(true); + var_dump(self::$tup2[0]); + + var_dump(self::$tup3[6]); + var_dump(self::$tup3[7]); + + var_dump(self::$tup4[7]); + var_dump(self::$tup4[8]); + } +} + +WithGlobals::use(); + +global $g_unknown; + +function accessUnknown() { + global $g_unknown; +} + +accessUnknown(); + +class WithUnusedMethod { + static public int $used_only_in_unreachable = 123; + function unreachableViaCfgMethod() { + echo self::$used_only_in_unreachable, "\n"; + } + function usedMethod() { + throw new Exception; + // this call will be deleted in cfg + $this->unreachableViaCfgMethod(); + } +} + +if (0) (new WithUnusedMethod)->usedMethod(); + +function useSuperglobals() { + $_REQUEST = ['l' => 1]; + echo "count _REQUEST = ", count($_REQUEST), "\n"; + echo "php_sapi = ", PHP_SAPI, "\n"; +} + +useSuperglobals(); + +function toLowerPrint() { + var_dump (mb_strtolower('ABCABC')); +} +toLowerPrint(); + +trait T { + static public int $i = 0; + + public function incAndPrint() { + self::$i++; + echo get_class($this), " = ", self::$i, "\n"; + } +} + +class TInst1 { + use T; +} + +(new TInst1)->incAndPrint(); + +class UnusedClass1 { + static public $field1 = 0; + static private $field2 = null; +} + +class UnusedClass2 { + use T; +} + +$global_arr = [1,2,3]; +foreach ($global_arr as &$global_item_ref) { + $global_item_ref *= 2; +} +unset($global_item_ref); +echo "global_arr = ", implode(',', $global_arr), "\n"; + +class H { function str() { return 'str'; } } +function heredoc(): H { return new H; } + +$html1 = 'div'; +$html2 = 'span'; +if (true) { + $hd = heredoc(); + $html1 .= " title=\"{$hd->str()}\""; + $html2 .= " header=\"{$hd->str()}\""; +} +echo $html1, ' ', $html2, "\n"; + +$htmls = ['i', 'b']; +if (true) { + $hd = heredoc(); + foreach ($htmls as &$htmli) { + $htmli .= " data-txt=\"{$hd->str()}\""; + } +} +echo implode(' ', $htmls), "\n"; + +class SomeAnother22 { + static public string $final_message = ''; + + static function formAndPrint() { + $owner_id = 1; + SomeAnother22::$final_message .= "owner_id = $owner_id"; + echo SomeAnother22::$final_message, "\n"; + } +} + +SomeAnother22::formAndPrint(); + +function declaresGlobalButNotUsesIt() { + global $g_unknown, $g_assigned_const_bool; + static $s_unused = 0; + echo __FUNCTION__, "\n"; + return; + echo $g_assigned_const_bool; + echo $s_unused; +} + +declaresGlobalButNotUsesIt(); + +class UsedInFunctionStaticOnly { + public string $str = ''; +} + +function hasStaticInstance() { + /** @var UsedInFunctionStaticOnly $inst */ + static $inst = null; + if ($inst === null) + $inst = new UsedInFunctionStaticOnly; +} +hasStaticInstance(); + diff --git a/tests/python/tests/ffi/test_ffi.py b/tests/python/tests/ffi/test_ffi.py index 9ce429a2c0..b40e013d8f 100644 --- a/tests/python/tests/ffi/test_ffi.py +++ b/tests/python/tests/ffi/test_ffi.py @@ -65,7 +65,7 @@ def codegen_find(self, patterns): full_name = os.path.join(path, f) if not os.path.isdir(full_name): continue - if f.startswith('o_vars_'): + if f.startswith('o_globals_') or f.startswith('o_const_'): continue for f2 in os.listdir(full_name): with open(os.path.join(full_name, f2), 'r') as cpp_file: diff --git a/tests/python/tests/libs/php/lib_examples/example1/php/index.php b/tests/python/tests/libs/php/lib_examples/example1/php/index.php index b5cc7c3150..e79bdb3562 100644 --- a/tests/python/tests/libs/php/lib_examples/example1/php/index.php +++ b/tests/python/tests/libs/php/lib_examples/example1/php/index.php @@ -1,12 +1,19 @@ Date: Tue, 18 Jun 2024 19:27:00 +0300 Subject: [PATCH 29/89] Added environment to `StatsHouse` and `engine.stat` (#1019) --- server/php-master.cpp | 6 +++++- server/server-config.cpp | 3 +++ server/server-config.h | 6 ++++++ server/statshouse/statshouse-client.h | 4 ++++ server/statshouse/statshouse-manager.cpp | 6 +++++- 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/server/php-master.cpp b/server/php-master.cpp index b0efe754ba..a7434a409c 100644 --- a/server/php-master.cpp +++ b/server/php-master.cpp @@ -960,7 +960,11 @@ std::string php_master_prepare_stats(bool add_worker_pids) { // engine_tag may be ended with "[" oss << "kphp_version\t" << atoll(engine_tag) << "\n"; } - oss << "cluster_name\t" << vk::singleton::get().get_cluster_name() << "\n" + const auto &config = vk::singleton::get(); + if (!config.get_environment().empty()) { + oss << "environment\t" << config.get_environment() << "\n"; + } + oss << "cluster_name\t" << config.get_cluster_name() << "\n" << "master_name\t" << vk::singleton::get().get_master_name() << "\n" << "min_worker_uptime\t" << min_uptime << "\n" << "max_worker_uptime\t" << max_uptime << "\n" diff --git a/server/server-config.cpp b/server/server-config.cpp index 88a0cfbf7d..42fd89cb42 100644 --- a/server/server-config.cpp +++ b/server/server-config.cpp @@ -55,6 +55,9 @@ int ServerConfig::init_from_config(const char *config_path) noexcept { if (auto err_msg = set_cluster_name(cluster_name.data(), false)) { throw std::runtime_error(err_msg); } + if (auto environment = node["environment"]) { + environment_ = environment.as(); + } } catch (const std::exception &e) { kprintf("--server-config, incorrect server config: '%s'\n%s\n", config_path, e.what()); return -1; diff --git a/server/server-config.h b/server/server-config.h index 6e160f3f83..55d8bd7ca5 100644 --- a/server/server-config.h +++ b/server/server-config.h @@ -6,6 +6,7 @@ #include #include +#include #include #include "common/mixin/not_copyable.h" @@ -21,6 +22,10 @@ class ServerConfig : vk::not_copyable { return statsd_prefix_.data(); } + const std::string &get_environment() const { + return environment_; + } + const char *set_cluster_name(const char *cluster_name, bool deprecated) noexcept; int init_from_config(const char *config_path) noexcept; @@ -34,4 +39,5 @@ class ServerConfig : vk::not_copyable { std::array cluster_name_; std::array statsd_prefix_; + std::string environment_; }; diff --git a/server/statshouse/statshouse-client.h b/server/statshouse/statshouse-client.h index e154cc037e..54f9761f38 100644 --- a/server/statshouse/statshouse-client.h +++ b/server/statshouse/statshouse-client.h @@ -15,6 +15,10 @@ class StatsHouseClient { statshouse::TransportUDPBase::MetricBuilder metric(std::string_view name, bool force_tag_host = false); + void set_environment(const std::string &env) { + transport.set_default_env(env); + } + void set_tag_cluster(std::string_view cluster) { tag_cluster = cluster; } diff --git a/server/statshouse/statshouse-manager.cpp b/server/statshouse/statshouse-manager.cpp index 8e4f36f2bd..9af11021d0 100644 --- a/server/statshouse/statshouse-manager.cpp +++ b/server/statshouse/statshouse-manager.cpp @@ -57,7 +57,11 @@ StatsHouseManager::StatsHouseManager(const std::string &ip, int port) : client(ip, port){}; void StatsHouseManager::set_common_tags() { - client.set_tag_cluster(vk::singleton::get().get_cluster_name()); + const auto &config = vk::singleton::get(); + if (!config.get_environment().empty()) { + client.set_environment(config.get_environment()); + } + client.set_tag_cluster(config.get_cluster_name()); client.set_tag_host(kdb_gethostname()); } From 76bc5779d3f8c1526edada72b5aa5b44ef15aa98 Mon Sep 17 00:00:00 2001 From: Leonid Znamenok Date: Fri, 17 May 2024 11:54:43 +0300 Subject: [PATCH 30/89] Set absolute path for VK_INSTALL_DIR --- cmake/init-global-vars.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/init-global-vars.cmake b/cmake/init-global-vars.cmake index e9787f5b8f..2111150e05 100644 --- a/cmake/init-global-vars.cmake +++ b/cmake/init-global-vars.cmake @@ -73,7 +73,7 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "/." CACHE PATH "install prefix" FORCE) endif() -set(VK_INSTALL_DIR usr/share/vkontakte) +set(VK_INSTALL_DIR /usr/share/vkontakte) set(INSTALL_KPHP_SOURCE ${VK_INSTALL_DIR}/kphp_source) set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) From d6308354e5ffbe893227ea25e69552f8d61bee73 Mon Sep 17 00:00:00 2001 From: Vladislav Senin Date: Wed, 19 Jun 2024 12:05:43 +0300 Subject: [PATCH 31/89] added cstdint header files (#1021) --- common/algorithms/hashes.h | 1 + common/algorithms/string-algorithms.h | 2 ++ compiler/compiler-settings.h | 2 ++ compiler/data/performance-inspections.h | 6 ++++-- compiler/ffi/ffi_types.h | 3 ++- compiler/inferring/multi-key.h | 1 + compiler/stage.h | 1 + 7 files changed, 13 insertions(+), 3 deletions(-) diff --git a/common/algorithms/hashes.h b/common/algorithms/hashes.h index 48d20f079e..6da073f881 100644 --- a/common/algorithms/hashes.h +++ b/common/algorithms/hashes.h @@ -6,6 +6,7 @@ #define ENGINE_HASHES_H #include +#include #include #include #include diff --git a/common/algorithms/string-algorithms.h b/common/algorithms/string-algorithms.h index 133dc5328b..ba6f062518 100644 --- a/common/algorithms/string-algorithms.h +++ b/common/algorithms/string-algorithms.h @@ -4,6 +4,8 @@ #pragma once +#include + #include "common/functional/identity.h" #include "common/smart_iterators/transform_iterator.h" #include "common/wrappers/string_view.h" diff --git a/compiler/compiler-settings.h b/compiler/compiler-settings.h index 11342b99e7..dc00a23c08 100644 --- a/compiler/compiler-settings.h +++ b/compiler/compiler-settings.h @@ -3,6 +3,8 @@ // Distributed under the GPL v3 License, see LICENSE.notice.txt #pragma once + +#include #include #include #include diff --git a/compiler/data/performance-inspections.h b/compiler/data/performance-inspections.h index 74f5e4423a..fef611a25f 100644 --- a/compiler/data/performance-inspections.h +++ b/compiler/data/performance-inspections.h @@ -3,9 +3,11 @@ // Distributed under the GPL v3 License, see LICENSE.notice.txt #pragma once -#include "compiler/data/data_ptr.h" -#include "common/wrappers/string_view.h" +#include + +#include "common/wrappers/string_view.h" +#include "compiler/data/data_ptr.h" class PerformanceInspections { public: diff --git a/compiler/ffi/ffi_types.h b/compiler/ffi/ffi_types.h index f4ef733cd0..146e763a3b 100644 --- a/compiler/ffi/ffi_types.h +++ b/compiler/ffi/ffi_types.h @@ -4,9 +4,10 @@ #pragma once +#include #include -#include #include +#include enum class FFITypeKind: uint16_t { Unknown, diff --git a/compiler/inferring/multi-key.h b/compiler/inferring/multi-key.h index 66219618f7..dd41bf2220 100644 --- a/compiler/inferring/multi-key.h +++ b/compiler/inferring/multi-key.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include diff --git a/compiler/stage.h b/compiler/stage.h index 7d40ddc894..9b14a947b0 100644 --- a/compiler/stage.h +++ b/compiler/stage.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include "compiler/data/data_ptr.h" From 3e2cdf624a6de7f95dfab2aa4a5074139fb624b6 Mon Sep 17 00:00:00 2001 From: Aleksandr Kirsanov Date: Wed, 19 Jun 2024 16:09:50 +0300 Subject: [PATCH 32/89] Fix compilation error for lambdas in unused functions (#1016) Previously, in certain cases, there could be a compilation error: > var Lambda$ufa6f526451637135_0::$tmp_var is declared > but never written; please, provide a default value --- compiler/pipes/check-classes.cpp | 1 - compiler/pipes/filter-only-actually-used.cpp | 82 ++++++++++++++++++++ compiler/pipes/generate-virtual-methods.cpp | 6 +- tests/phpt/lambdas/020_uses_in_lambda.php | 61 +++++++++++++++ 4 files changed, 144 insertions(+), 6 deletions(-) diff --git a/compiler/pipes/check-classes.cpp b/compiler/pipes/check-classes.cpp index 8c788019ae..75e5765129 100644 --- a/compiler/pipes/check-classes.cpp +++ b/compiler/pipes/check-classes.cpp @@ -72,7 +72,6 @@ inline void CheckClassesPass::check_static_fields_inited(ClassPtr klass) { } inline void CheckClassesPass::check_instance_fields_inited(ClassPtr klass) { - // TODO KPHP-221: the old code is kept for now (check for Unknown) klass->members.for_each([](const ClassMemberInstanceField &f) { PrimitiveType ptype = f.var->tinf_node.get_type()->get_real_ptype(); kphp_error(ptype != tp_any, diff --git a/compiler/pipes/filter-only-actually-used.cpp b/compiler/pipes/filter-only-actually-used.cpp index 4c77e8eaf8..8cfbdf905d 100644 --- a/compiler/pipes/filter-only-actually-used.cpp +++ b/compiler/pipes/filter-only-actually-used.cpp @@ -9,6 +9,78 @@ #include "compiler/data/src-file.h" #include "compiler/compiler-core.h" #include "compiler/threading/profiler.h" +#include "compiler/vertex-util.h" + +// having a typed callable __invoke(), which is a virtual function with switch-case dispatching, +// replace body of `case {lambda_class_to_remove.hash()}:` (which is a lambda invoke call) +// to just `break`; see below, why this is important +class RemoveLambdaCallFromTypedCallablePass final : public FunctionPassBase { + std::string case_hash; + +public: + std::string get_description() override { + return "Remove lambda call from typed callable"; + } + + explicit RemoveLambdaCallFromTypedCallablePass(ClassPtr lambda_class_to_remove) + : case_hash(std::to_string(lambda_class_to_remove->get_hash())) {} + + VertexPtr on_enter_vertex(VertexPtr root) override { + if (auto as_case = root.try_as()) { + if (auto as_int_const = as_case->expr().try_as()) { + if (as_int_const->str_val == case_hash) { + auto level1 = VertexUtil::create_int_const(1); + return VertexAdaptor::create(as_int_const, VertexAdaptor::create(VertexAdaptor::create(level1))); + } + } + } + + return root; + } +}; + +// when a lambda with `use` statement (=> with an instance field) occurs inside unused function, +// it still can be reachable from a typed callable __invoke(), +// but its field types can't be inferred, they'll be left 'any' and trigger an error after +// to prevent this, manually remove this lambda from that __invoke() body +// as well as remove all lambda's mentions from used_functions +class AnalyzeLambdasInUnusedFunctionPass final : public FunctionPassBase { + IdMap &used_functions; + +public: + std::string get_description() override { + return "Analyze lambdas in unused function"; + } + + explicit AnalyzeLambdasInUnusedFunctionPass(IdMap &used_functions) + : used_functions(used_functions) {} + + VertexPtr on_enter_vertex(VertexPtr root) override { + if (auto as_call = root.try_as(); + as_call && as_call->func_id->class_id && as_call->func_id->class_id->is_lambda_class() && as_call->func_id->is_constructor()) { + ClassPtr lambda_class = as_call->func_id->class_id; + const ClassMemberInstanceMethod *m_invoke = lambda_class->members.get_instance_method("__invoke"); + if (m_invoke && used_functions[m_invoke->function->outer_function]) { + // f_lambda occurs inside unused function, but is used; the only reason is it's used from a typed callable + FunctionPtr f_lambda = m_invoke->function->outer_function; + kphp_assert(lambda_class->implements.size() == 1 && lambda_class->implements[0]->is_typed_callable_interface()); + FunctionPtr f_typed_invoke = lambda_class->implements[0]->members.get_instance_method("__invoke")->function; + + RemoveLambdaCallFromTypedCallablePass pass(lambda_class); + run_function_pass(f_typed_invoke->root, &pass); + + AnalyzeLambdasInUnusedFunctionPass self_pass(used_functions); + run_function_pass(f_lambda, &self_pass); + + used_functions[f_lambda] = {}; + used_functions[m_invoke->function] = {}; + used_functions[lambda_class->construct_function] = {}; + } + } + + return root; + } +}; namespace { @@ -261,6 +333,16 @@ void FilterOnlyActuallyUsedFunctionsF::on_finish(DataStream &os) { remove_unused_class_methods(all, used_functions); stage::die_if_global_errors(); + // remove lambdas from unused functions, see comments above + for (const auto &f_and_e : all) { + FunctionPtr fun = f_and_e.first; + if (fun->has_lambdas_inside && !fun->is_lambda() && !used_functions[fun]) { + AnalyzeLambdasInUnusedFunctionPass pass(used_functions); + run_function_pass(fun, &pass); + } + } + stage::die_if_global_errors(); + // forward the reachable functions into the data stream; // this should be the last step for (const auto &f : used_functions) { diff --git a/compiler/pipes/generate-virtual-methods.cpp b/compiler/pipes/generate-virtual-methods.cpp index 681b564473..5bb1e2b4ce 100644 --- a/compiler/pipes/generate-virtual-methods.cpp +++ b/compiler/pipes/generate-virtual-methods.cpp @@ -422,10 +422,6 @@ void generate_body_of_virtual_method(FunctionPtr virtual_function) { cases.emplace_back(v_case); } } - if (!cases.empty()) { - auto case_default_warn = generate_critical_error_call(fmt_format("call method({}) on null object", virtual_function->as_human_readable(false))); - cases.emplace_back(VertexAdaptor::create(VertexAdaptor::create(case_default_warn))); - } if (cases.empty() && !stage::has_error()) { // when there are no inheritors of an interface, generate an empty body if possible — @@ -436,7 +432,7 @@ void generate_body_of_virtual_method(FunctionPtr virtual_function) { auto call_get_hash = VertexAdaptor::create(ClassData::gen_vertex_this({})); call_get_hash->str_val = "get_hash_of_class"; call_get_hash->func_id = G->get_function(call_get_hash->str_val); - virtual_function->root->cmd_ref() = VertexAdaptor::create(VertexUtil::create_switch_vertex(virtual_function, call_get_hash, std::move(cases))); + virtual_function->root->cmd_ref() = VertexAdaptor::create(VertexUtil::create_switch_vertex(virtual_function, call_get_hash, std::move(cases)), generate_critical_error_call(fmt_format("call method({}) on null object", virtual_function->as_human_readable(false)))); } virtual_function->type = FunctionData::func_local; // could be func_extern before, but now it has a body diff --git a/tests/phpt/lambdas/020_uses_in_lambda.php b/tests/phpt/lambdas/020_uses_in_lambda.php index cb185054a3..4dff4c2f53 100644 --- a/tests/phpt/lambdas/020_uses_in_lambda.php +++ b/tests/phpt/lambdas/020_uses_in_lambda.php @@ -110,3 +110,64 @@ function l2_modif() { } (new WithFCapturingDeep)->l2(); (new WithFCapturingDeep)->l2_modif(); + +class Example020 { + static private function takeInt(int $i) { echo $i; } + + public function unused_function(): void { + $tmp_var = ""; + $this->call_function(function() use ($tmp_var): void {}); + $int = 10; + $this->call_function(function() use ($int): void { self::takeInt($int); }); + } + + /** @param callable():void $fn */ + public function call_function(callable $fn): void { + $fn(); + } +} + +(new Example020())->call_function(function(): void {}); + +class Bxample020 { + /** + * @param callable(int):int $fn + */ + public function unused_function(callable $fn): int { + return $this->test2(function () use ($fn) { + return $fn(1); + }); + } + + /** + * @param callable():int $fn + */ + public function test2(callable $fn): int { + return $fn(); + } +} + +$bxample = new Bxample020(); +$bxample->test2(function() { return 0; }); + + +class Cxample { + public function unused_function(): void { + $tmp_var = ""; + $this->call_function_first(function() use ($tmp_var): void { + $this->call_function_second(function() use ($tmp_var) {}); + }); + } + + /** @param callable():void $fn */ + public function call_function_first(callable $fn): void { + $fn(); + } + + /** @param callable():void $fn */ + public function call_function_second(callable $fn): void { + $fn(); + } +} + +(new Cxample)->call_function_first(function():void{} ); From 7fb45472743603ee429c32a2a06db6ba159032b4 Mon Sep 17 00:00:00 2001 From: Alexander Polyakov Date: Wed, 19 Jun 2024 17:47:15 +0300 Subject: [PATCH 33/89] Engines common headers support (#1000) This commit adds support for the new format of engine common headers. --- common/binlog/binlog-snapshot.cpp | 73 ++++++++++ common/binlog/binlog-snapshot.h | 59 ++++++++ common/binlog/binlog.cmake | 3 +- common/binlog/snapshot-shifts.h | 42 +++--- server/confdata-binlog-replay.cpp | 204 ++++++++++++++++++++++----- server/pmemcached-binlog-interface.h | 3 - 6 files changed, 330 insertions(+), 54 deletions(-) create mode 100644 common/binlog/binlog-snapshot.cpp create mode 100644 common/binlog/binlog-snapshot.h diff --git a/common/binlog/binlog-snapshot.cpp b/common/binlog/binlog-snapshot.cpp new file mode 100644 index 0000000000..accaf501bc --- /dev/null +++ b/common/binlog/binlog-snapshot.cpp @@ -0,0 +1,73 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "common/binlog/binlog-snapshot.h" + +#include + +#include "common/tl/fetch.h" + +namespace kphp { +namespace tl { + +namespace { + +constexpr int32_t RESULT_TRUE_MAGIC{0x3f9c8ef8}; + +} // namespace + +BarsicSnapshotHeader::BarsicSnapshotHeader() + : fields_mask() + , dependencies(DEPENDENCIES_BUFFER_SIZE) + , payload_offset() {} + +void BarsicSnapshotHeader::SnapshotDependency::tl_fetch() noexcept { + std::basic_string buffer{}; + buffer.reserve(STRING_BUFFER_SIZE); + + fields_mask = tl_fetch_int(); + // skip cluster_id + vk::tl::fetch_string(buffer); + // skip shard_id + vk::tl::fetch_string(buffer); + payload_offset = tl_fetch_long(); +} + +void BarsicSnapshotHeader::tl_fetch() noexcept { + std::basic_string buffer{}; + buffer.reserve(STRING_BUFFER_SIZE); + + fields_mask = tl_fetch_int(); + // skip cluster_id + vk::tl::fetch_string(buffer); + // skip shard_id + vk::tl::fetch_string(buffer); + // skip snapshot_meta + vk::tl::fetch_string(buffer); + // skip dependencies + vk::tl::fetch_vector(dependencies); + + payload_offset = tl_fetch_long(); + + // skip engine_version + vk::tl::fetch_string(buffer); + // skip creation_time_nano + std::ignore = tl_fetch_long(); + // skip control_meta + if (static_cast(fields_mask & 0x1)) { + vk::tl::fetch_string(buffer); + } +} + +void TlEngineSnapshotHeader::tl_fetch() noexcept { + fields_mask = tl_fetch_int(); + binlog_time_sec = tl_fetch_long(); + + if (tl_fetch_int() == RESULT_TRUE_MAGIC) { + file_binlog_crc = tl_fetch_int(); + } +} + +} // namespace tl +} // namespace kphp diff --git a/common/binlog/binlog-snapshot.h b/common/binlog/binlog-snapshot.h new file mode 100644 index 0000000000..6cf9c99510 --- /dev/null +++ b/common/binlog/binlog-snapshot.h @@ -0,0 +1,59 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include + +namespace kphp { +namespace tl { + +constexpr auto UNEXPECTED_TL_MAGIC_ERROR_FORMAT = "unexpected TL magic 0x%x, expected 0x%x\n"; + +constexpr auto COMMON_HEADER_META_SIZE = sizeof(int32_t) + sizeof(int64_t); +constexpr auto COMMON_HEADER_HASH_SIZE = 2 * sizeof(int64_t); + +constexpr int32_t PMEMCACHED_OLD_INDEX_MAGIC = 0x53407fa0; +constexpr int32_t PMEMCACHED_INDEX_RAM_MAGIC_G3 = 0x65049e9e; +constexpr int32_t BARSIC_SNAPSHOT_HEADER_MAGIC = 0x1d0d1b74; +constexpr int32_t TL_ENGINE_SNAPSHOT_HEADER_MAGIC = 0x4bf8b614; +constexpr int32_t PERSISTENT_CONFIG_V2_SNAPSHOT_BLOCK = 0x501096b7; +constexpr int32_t RPC_QUERIES_SNAPSHOT_QUERY_COMMON = 0x9586c501; +constexpr int32_t SNAPSHOT_MAGIC = 0xf0ec39fb; +constexpr int32_t COMMON_INFO_END = 0x5a9ce5ec; + +struct BarsicSnapshotHeader { + struct SnapshotDependency { + int32_t fields_mask; + int64_t payload_offset; + + void tl_fetch() noexcept; + }; + + int32_t fields_mask; + std::vector dependencies; + int64_t payload_offset; + + void tl_fetch() noexcept; + + BarsicSnapshotHeader(); + +private: + static constexpr auto STRING_BUFFER_SIZE = 512; + static constexpr auto DEPENDENCIES_BUFFER_SIZE = 128; +}; + +struct TlEngineSnapshotHeader { + int32_t fields_mask{}; + int64_t binlog_time_sec{}; + std::optional file_binlog_crc; + + void tl_fetch() noexcept; +}; + +} // namespace tl +} // namespace kphp diff --git a/common/binlog/binlog.cmake b/common/binlog/binlog.cmake index 7fa75452fd..1a6ecbadb9 100644 --- a/common/binlog/binlog.cmake +++ b/common/binlog/binlog.cmake @@ -3,6 +3,7 @@ prepend(BINLOG_SOURCES ${COMMON_DIR}/binlog/ binlog-buffer.cpp binlog-buffer-aio.cpp binlog-buffer-rotation-points.cpp - binlog-buffer-replay.cpp) + binlog-buffer-replay.cpp + binlog-snapshot.cpp) vk_add_library(binlog_src OBJECT ${BINLOG_SOURCES}) diff --git a/common/binlog/snapshot-shifts.h b/common/binlog/snapshot-shifts.h index 5a28872a4c..2bd0c3d0f6 100644 --- a/common/binlog/snapshot-shifts.h +++ b/common/binlog/snapshot-shifts.h @@ -4,28 +4,34 @@ #pragma once -inline static int get_snapshot_position_shift(const struct kfs_file_info *info) { +#include + +#include "common/binlog/binlog-snapshot.h" +#include "common/tl/methods/string.h" + +inline static long long get_snapshot_log_pos(const struct kfs_file_info *info) { + long long log_pos{-1}; if (info->preloaded_bytes < 4) { - return 0xffff; + return log_pos; } - int magic = *(int *)(info->start); - if (magic == 0x53407fa0) { // PMEMCACHED_RAM_INDEX_MAGIC - return 16; - } - fprintf(stderr, "Unknown snapshot magic for file %s: %08x\n", info->filename, magic); - return 0xffff; -} -inline static long long get_snapshot_log_pos(const struct kfs_file_info *info) { - int shift = get_snapshot_position_shift(info); - long long log_pos = -1; - if (info->preloaded_bytes >= shift + 8) { - log_pos = *(long long *)(info->start + shift); - if (!(info->min_log_pos <= log_pos && log_pos <= info->max_log_pos)) { - fprintf(stderr, "filename %s info->min_log_pos %lld info->max_log_pos %lld log_pos %lld shift %d\n", info->filename, info->min_log_pos, info->max_log_pos, log_pos, shift); - assert(info->min_log_pos <= log_pos && log_pos <= info->max_log_pos); + const auto magic{*reinterpret_cast(info->start)}; + if (magic == kphp::tl::PMEMCACHED_OLD_INDEX_MAGIC) { + log_pos = *reinterpret_cast(info->start + 2 * sizeof(int32_t) + sizeof(int64_t)); // add offset of log_pos1 + } else if (magic == kphp::tl::BARSIC_SNAPSHOT_HEADER_MAGIC && info->preloaded_bytes >= kphp::tl::COMMON_HEADER_META_SIZE - sizeof(int32_t)) { + const auto tl_body_len{*reinterpret_cast(info->start + sizeof(int32_t))}; + if (info->preloaded_bytes >= kphp::tl::COMMON_HEADER_META_SIZE + tl_body_len) { + kphp::tl::BarsicSnapshotHeader bsh{}; + vk::tl::fetch_from_buffer(info->start + kphp::tl::COMMON_HEADER_META_SIZE, tl_body_len, bsh); + log_pos = bsh.payload_offset; } + } else { + fprintf(stderr, "Unknown snapshot magic for file %s: %08x\n", info->filename, magic); + } + + if (log_pos < info->min_log_pos || log_pos > info->max_log_pos) { + fprintf(stderr, "filename %s info->min_log_pos %lld info->max_log_pos %lld log_pos %lld\n", info->filename, info->min_log_pos, info->max_log_pos, log_pos); } + return log_pos; } - diff --git a/server/confdata-binlog-replay.cpp b/server/confdata-binlog-replay.cpp index c8994c6f2e..7f4c6b58ce 100644 --- a/server/confdata-binlog-replay.cpp +++ b/server/confdata-binlog-replay.cpp @@ -4,26 +4,28 @@ #include "server/confdata-binlog-replay.h" +#include #include #include #include #include +#include #include #include #include +#include +#include #include "common/binlog/binlog-replayer.h" +#include "common/binlog/binlog-snapshot.h" #include "common/dl-utils-lite.h" -#include "common/precise-time.h" +#include "common/kfs/kfs.h" #include "common/server/engine-settings.h" #include "common/server/init-binlog.h" #include "common/server/init-snapshot.h" +#include "common/tl/methods/string.h" #include "common/wrappers/string_view.h" -#include "common/kfs/kfs.h" - -#include "runtime/allocator.h" #include "runtime/confdata-global-manager.h" -#include "runtime/kphp_core.h" #include "server/confdata-binlog-events.h" #include "server/confdata-stats.h" #include "server/server-log.h" @@ -116,34 +118,16 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { return dot_pos; } - int load_index() noexcept { - if (!Snapshot) { - jump_log_ts = 0; - jump_log_pos = 0; - jump_log_crc32 = 0; - return 0; - } - index_header header; - kfs_read_file_assert (Snapshot, &header, sizeof(index_header)); - if (header.magic != PMEMCACHED_INDEX_MAGIC) { - fprintf(stderr, "index file is not for confdata\n"); - return -1; - } - jump_log_ts = header.log_timestamp; - jump_log_pos = header.log_pos1; - jump_log_crc32 = header.log_pos1_crc32; - - const int nrecords = header.nrecords; - vkprintf(2, "%d records readed\n", nrecords); - auto index_offset = std::make_unique(nrecords + 1); + int process_confdata_snapshot_entries(index_header &header) noexcept { + const auto index_offset = std::make_unique(header.nrecords + 1); assert (index_offset); - kfs_read_file_assert (Snapshot, index_offset.get(), sizeof(index_offset[0]) * (nrecords + 1)); - vkprintf(1, "index_offset[%d]=%" PRId64 "\n", nrecords, index_offset[nrecords]); + kfs_read_file_assert(Snapshot, index_offset.get(), sizeof(index_offset[0]) * (header.nrecords + 1)); + vkprintf(1, "index_offset[%d]=%" PRId64 "\n", header.nrecords, index_offset[header.nrecords]); - auto index_binary_data = std::make_unique(index_offset[nrecords]); + const auto index_binary_data = std::make_unique(index_offset[header.nrecords]); assert (index_binary_data); - kfs_read_file_assert (Snapshot, index_binary_data.get(), index_offset[nrecords]); + kfs_read_file_assert(Snapshot, index_binary_data.get(), index_offset[header.nrecords]); using entry_type = lev_confdata_store_wrapper; @@ -151,7 +135,7 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { vk::string_view last_two_dots_key; array_size one_dot_elements_counter; array_size two_dots_elements_counter; - for (int i = 0; i < nrecords; i++) { + for (auto i = 0; i < header.nrecords; i++) { const auto &element = reinterpret_cast(index_binary_data[index_offset[i]]); const vk::string_view key{element.data, static_cast(std::max(element.key_len, short{0}))}; if (key.empty() || key_blacklist_.is_blacklisted(key)) { @@ -168,7 +152,7 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { // disable the blacklist because we checked the keys during the previous step blacklist_enabled_ = false; - for (int i = 0; i < nrecords; i++) { + for (auto i = 0; i < header.nrecords; i++) { if (index_offset[i] >= 0) { store_element(reinterpret_cast(index_binary_data[index_offset[i]])); } @@ -184,6 +168,161 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { return 0; } + int process_old_confdata_snapshot() noexcept { + index_header header; + kfs_read_file_assert(Snapshot, &header, sizeof(index_header)); + + if (header.magic != kphp::tl::PMEMCACHED_OLD_INDEX_MAGIC) { + fprintf(stderr, kphp::tl::UNEXPECTED_TL_MAGIC_ERROR_FORMAT, header.magic, kphp::tl::PMEMCACHED_OLD_INDEX_MAGIC); + return -1; + } + + jump_log_ts = header.log_timestamp; + jump_log_pos = header.log_pos1; + jump_log_crc32 = header.log_pos1_crc32; + + return process_confdata_snapshot_entries(header); + } + + int process_barsic_common_header() noexcept { + std::array header_meta{}; + kfs_read_file_assert(Snapshot, header_meta.data(), header_meta.size()); + + int32_t magic{}; + vk::tl::fetch_from_buffer(reinterpret_cast(header_meta.data()), header_meta.size(), magic); + if (magic != kphp::tl::BARSIC_SNAPSHOT_HEADER_MAGIC) { + fprintf(stderr, kphp::tl::UNEXPECTED_TL_MAGIC_ERROR_FORMAT, magic, kphp::tl::BARSIC_SNAPSHOT_HEADER_MAGIC); + return -1; + } + + int64_t tl_body_len{}; + vk::tl::fetch_from_buffer(reinterpret_cast(header_meta.data() + sizeof(int32_t)), header_meta.size() - sizeof(int32_t), + tl_body_len); + + std::vector buffer{static_cast(tl_body_len + kphp::tl::COMMON_HEADER_HASH_SIZE)}; + kfs_read_file_assert(Snapshot, buffer.data(), buffer.size()); + + kphp::tl::BarsicSnapshotHeader bsh{}; + vk::tl::fetch_from_buffer(reinterpret_cast(buffer.data()), buffer.size(), bsh); + // TODO: compute xxhash + + jump_log_pos = bsh.payload_offset; + return 0; + } + + int process_confdata_engine_header() noexcept { + std::array header_meta{}; + kfs_read_file_assert(Snapshot, header_meta.data(), header_meta.size()); + + int32_t magic{}; + vk::tl::fetch_from_buffer(reinterpret_cast(header_meta.data()), header_meta.size(), magic); + if (magic != kphp::tl::TL_ENGINE_SNAPSHOT_HEADER_MAGIC) { + fprintf(stderr, kphp::tl::UNEXPECTED_TL_MAGIC_ERROR_FORMAT, magic, kphp::tl::TL_ENGINE_SNAPSHOT_HEADER_MAGIC); + return -1; + } + + int64_t tl_body_len{}; + vk::tl::fetch_from_buffer(reinterpret_cast(header_meta.data() + sizeof(int32_t)), header_meta.size() - sizeof(int32_t), + tl_body_len); + + std::vector buffer{static_cast(tl_body_len + kphp::tl::COMMON_HEADER_HASH_SIZE)}; + kfs_read_file_assert(Snapshot, buffer.data(), buffer.size()); + + kphp::tl::TlEngineSnapshotHeader esh{}; + vk::tl::fetch_from_buffer(reinterpret_cast(buffer.data()), buffer.size(), esh); + // TODO: compute xxhash + + jump_log_ts = esh.binlog_time_sec; + jump_log_crc32 = esh.file_binlog_crc.value_or(0); + return 0; + } + + int skip_persistent_config() noexcept { + int32_t magic{}; + kfs_read_file_assert(Snapshot, &magic, sizeof(int32_t)); + if (magic != kphp::tl::PERSISTENT_CONFIG_V2_SNAPSHOT_BLOCK) { + fprintf(stderr, kphp::tl::UNEXPECTED_TL_MAGIC_ERROR_FORMAT, magic, kphp::tl::PERSISTENT_CONFIG_V2_SNAPSHOT_BLOCK); + return -1; + } + + int32_t num_sizes{}; + kfs_read_file_assert(Snapshot, &num_sizes, sizeof(int32_t)); + + std::vector sizes(num_sizes); + kfs_read_file_assert(Snapshot, reinterpret_cast(sizes.data()), sizes.size() * sizeof(int64_t)); + + // use sizeof(int32_t) as initial value for accumulate since persistent config ends with crc32 + ::lseek(Snapshot->fd, std::accumulate(sizes.cbegin(), sizes.cend(), static_cast(sizeof(int32_t))), SEEK_CUR); + return 0; + } + + int skip_persistent_queries() noexcept { + int32_t size{}; + kfs_read_file_assert(Snapshot, &size, sizeof(int32_t)); + if (size != 0) { + fprintf(stderr, "unexpected non-zero size of persistent queries: %d\n", size); + return -1; + } + // skip hash + ::lseek(Snapshot->fd, kphp::tl::COMMON_HEADER_HASH_SIZE, SEEK_CUR); + return 0; + } + + int process_snapshot() noexcept { + if (!Snapshot) { + jump_log_ts = 0; + jump_log_pos = 0; + jump_log_crc32 = 0; + return 0; + } + + int32_t header_magic{}; + kfs_read_file_assert(Snapshot, &header_magic, sizeof(int32_t)); + // move cursor back so old index reader can safely read a header it expects + ::lseek(Snapshot->fd, -sizeof(int32_t), SEEK_CUR); + + if (header_magic == kphp::tl::PMEMCACHED_OLD_INDEX_MAGIC) { + return process_old_confdata_snapshot(); + } else if (header_magic == kphp::tl::BARSIC_SNAPSHOT_HEADER_MAGIC) { + if (process_barsic_common_header() != 0) { return -1; } + if (process_confdata_engine_header() != 0) { return -1; } + if (skip_persistent_config() != 0) { return -1; } + + kfs_read_file_assert(Snapshot, &header_magic, sizeof(int32_t)); + if (header_magic == kphp::tl::RPC_QUERIES_SNAPSHOT_QUERY_COMMON) { + // PMC code may write persistent query, but confdata should not have any + fprintf(stderr, "active persistent query (magic 0x%x) are not supported in confdata snapshots", kphp::tl::RPC_QUERIES_SNAPSHOT_QUERY_COMMON); + return -1; + } + + // engine code always writes this section, but it doesn't make any sense for confdata. + // Don't do strict check in case this section disappears + if (header_magic == kphp::tl::SNAPSHOT_MAGIC) { + if (skip_persistent_queries() != 0) { return -1; } + } else { + // move cursor back to let the next read take the whole index_header. + ::lseek(Snapshot->fd, -sizeof(int32_t), SEEK_CUR); + } + + kfs_read_file_assert(Snapshot, &header_magic, sizeof(int32_t)); + if (header_magic != kphp::tl::COMMON_INFO_END) { + fprintf(stderr, kphp::tl::UNEXPECTED_TL_MAGIC_ERROR_FORMAT, header_magic, kphp::tl::COMMON_INFO_END); + return -1; + } + + index_header idx_header{}; + kfs_read_file_assert(Snapshot, &idx_header, sizeof(index_header)); + if (idx_header.magic != kphp::tl::PMEMCACHED_INDEX_RAM_MAGIC_G3) { + fprintf(stderr, kphp::tl::UNEXPECTED_TL_MAGIC_ERROR_FORMAT, idx_header.magic, kphp::tl::PMEMCACHED_INDEX_RAM_MAGIC_G3); + return -1; + } + return process_confdata_snapshot_entries(idx_header); + } + + fprintf(stderr, "unexpected header magic: 0x%x\n", header_magic); + return -1; + } + OperationStatus delete_element(const char *key, short key_len) noexcept { auto memory_status = current_memory_status(); return generic_operation(key, key_len, -1, memory_status, [this] (MemoryStatus memory_status) { @@ -899,7 +1038,7 @@ void init_confdata_binlog_reader() noexcept { static engine_settings_t settings = {}; settings.name = NAME_VERSION; settings.load_index = []() { - return ConfdataBinlogReplayer::get().load_index(); + return ConfdataBinlogReplayer::get().process_snapshot(); }; settings.replay_logevent = [](const lev_generic *E, int size) { return ConfdataBinlogReplayer::get().replay(E, size); @@ -934,6 +1073,7 @@ void init_confdata_binlog_reader() noexcept { auto &confdata_binlog_replayer = ConfdataBinlogReplayer::get(); confdata_binlog_replayer.init(confdata_manager.get_resource()); engine_default_load_index(confdata_settings.binlog_mask); + update_confdata_state_from_binlog(true, 10 * confdata_settings.confdata_update_timeout_sec); if (confdata_binlog_replayer.current_memory_status() != ConfdataBinlogReplayer::MemoryStatus::NORMAL) { confdata_binlog_replayer.raise_confdata_oom_error("Can't read confdata binlog on start"); diff --git a/server/pmemcached-binlog-interface.h b/server/pmemcached-binlog-interface.h index 3fb778f4e7..fd9319757c 100644 --- a/server/pmemcached-binlog-interface.h +++ b/server/pmemcached-binlog-interface.h @@ -94,8 +94,6 @@ struct lev_pmemcached_touch { #pragma pack(pop) -#define PMEMCACHED_INDEX_MAGIC 0x53407fa0 - // snapshot structures typedef struct { /* strange numbers */ @@ -107,7 +105,6 @@ typedef struct { int log_timestamp; unsigned int log_pos0_crc32; unsigned int log_pos1_crc32; - int nrecords; } index_header; From a48153485dc5e29c1d646d2167d85f6953acc718 Mon Sep 17 00:00:00 2001 From: Denis Vaksman Date: Wed, 19 Jun 2024 17:48:04 +0300 Subject: [PATCH 34/89] cache uname syscall invocation (#1020) --- common/kernel-version.cpp | 4 +--- common/kernel-version.h | 3 ++- runtime/files.cpp | 9 ++++----- server/php-engine.cpp | 2 ++ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/common/kernel-version.cpp b/common/kernel-version.cpp index a9aa1accaa..c6173c7034 100644 --- a/common/kernel-version.cpp +++ b/common/kernel-version.cpp @@ -4,12 +4,10 @@ #include "common/kernel-version.h" -#include - #include "common/kprintf.h" #include "common/stats/provider.h" -static struct utsname* cached_uname() { +utsname* cached_uname() { static struct utsname kernel_version; static int got_kernel_version = 0; if (got_kernel_version == 0) { diff --git a/common/kernel-version.h b/common/kernel-version.h index 48012169b8..754c60f68c 100644 --- a/common/kernel-version.h +++ b/common/kernel-version.h @@ -5,7 +5,8 @@ #pragma once #include +#include +utsname* cached_uname(); int epoll_exclusive_supported(); int madvise_madv_free_supported(); - diff --git a/runtime/files.cpp b/runtime/files.cpp index e695734553..01f38ffa83 100644 --- a/runtime/files.cpp +++ b/runtime/files.cpp @@ -13,6 +13,7 @@ #undef basename +#include "common/kernel-version.h" #include "common/macos-ports.h" #include "common/wrappers/mkdir_recursive.h" @@ -361,13 +362,11 @@ bool f$mkdir(const string &name, int64_t mode, bool recursive) { } string f$php_uname(const string &name) { - utsname res; - dl::enter_critical_section();//OK - if (uname(&res)) { - dl::leave_critical_section(); + const auto *uname = cached_uname(); + if (uname == nullptr) { return {}; } - dl::leave_critical_section(); + const auto &res = *uname; char mode = name[0]; switch (mode) { diff --git a/server/php-engine.cpp b/server/php-engine.cpp index 3c2a1da794..61d1ce3bf1 100644 --- a/server/php-engine.cpp +++ b/server/php-engine.cpp @@ -26,6 +26,7 @@ #include "common/crc32c.h" #include "common/cycleclock.h" #include "common/dl-utils-lite.h" +#include "common/kernel-version.h" #include "common/kprintf.h" #include "common/macos-ports.h" #include "common/options.h" @@ -1667,6 +1668,7 @@ void init_all() { log_server_warning(deprecation_warning); } StatsHouseManager::get().set_common_tags(); + cached_uname(); // invoke uname syscall only once on master start global_init_runtime_libs(); init_php_scripts_once_in_master(); From 718abe481a1d24dbef8d3ae61181eedddea13332 Mon Sep 17 00:00:00 2001 From: Vladislav Senin Date: Fri, 21 Jun 2024 15:05:08 +0300 Subject: [PATCH 35/89] fixed cmake install for tlo-parsing (#1023) --- common/tlo-parsing/tlo-parsing.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/tlo-parsing/tlo-parsing.cmake b/common/tlo-parsing/tlo-parsing.cmake index 34017891a8..f03b70985a 100644 --- a/common/tlo-parsing/tlo-parsing.cmake +++ b/common/tlo-parsing/tlo-parsing.cmake @@ -26,7 +26,7 @@ set_target_properties(tlo_parsing_static PROPERTIES install(TARGETS tlo_parsing_static COMPONENT TLO_PARSING_DEV ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION /usr/${CMAKE_INSTALL_INCLUDEDIR}/tlo-parsing) + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/tlo-parsing) vk_add_library(tlo_parsing_shared SHARED $) set_target_properties(tlo_parsing_shared PROPERTIES OUTPUT_NAME tlo_parsing) From 39d905a1ca9931b9799853b567c97ee03bfc9ccd Mon Sep 17 00:00:00 2001 From: pale-emperor <46090986+pale-emperor@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:46:46 +0300 Subject: [PATCH 36/89] Create CODEOWNERS --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..2420214362 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +.github/ @VKCOM/vk-sec From 91a1f28440194c7131e1fe0ee061b9054f3b69d7 Mon Sep 17 00:00:00 2001 From: Vadim Sadokhov <65451602+astrophysik@users.noreply.github.com> Date: Wed, 26 Jun 2024 17:55:53 +0300 Subject: [PATCH 37/89] separate runtime to core and base parts (#1006) --- CMakeLists.txt | 1 + builtin-functions/_functions.txt | 2 +- common/stats/provider.h | 1 + .../contributing-to-kphp.md | 2 +- .../allocator/script-allocator-managed.h | 27 ++ .../class-instance/class-instance-decl.inl | 2 +- .../class-instance/class-instance.inl | 5 +- .../class-instance/refcountable-php-classes.h | 8 +- .../core-types}/comparison_operators.inl | 8 +- .../core-types}/conversions_types.inl | 2 +- .../core-types/decl}/array_decl.inl | 6 +- .../core-types/decl}/array_iterator.h | 2 +- .../core-types/decl}/declarations.h | 0 .../core-types/decl}/mixed_decl.inl | 2 +- .../core-types/decl}/optional.h | 4 +- .../core-types/decl}/shape.h | 0 .../core-types/decl}/string_buffer_decl.inl | 14 +- .../core-types/decl}/string_decl.inl | 2 +- .../core-types/definition}/array.inl | 18 +- .../core-types/definition}/mixed.cpp | 2 +- .../core-types/definition}/mixed.inl | 8 +- .../core-types/definition}/string.cpp | 2 +- .../core-types/definition}/string.inl | 16 +- .../core-types/definition/string_buffer.cpp | 15 + .../core-types/definition}/string_buffer.inl | 35 +-- .../core-types/definition}/string_cache.cpp | 2 +- .../core-types/definition}/string_cache.h | 2 +- .../core-types}/kphp_type_traits.h | 6 +- {runtime => runtime-core}/include.h | 8 +- .../details/memory_chunk_list.h | 2 +- .../details/memory_chunk_tree.cpp | 6 +- .../details/memory_chunk_tree.h | 2 +- .../details/memory_ordered_chunk_list.cpp | 2 +- .../details/memory_ordered_chunk_list.h | 4 +- .../details/universal_reallocate.h | 2 +- .../memory-resource}/extra-memory-pool.h | 0 .../memory-resource}/memory_resource.h | 4 - .../monotonic_buffer_resource.cpp | 19 ++ .../monotonic_buffer_resource.h | 9 +- .../memory-resource}/resource_allocator.h | 2 +- .../unsynchronized_pool_resource.cpp | 4 +- .../unsynchronized_pool_resource.h | 14 +- runtime-core/runtime-core-context.h | 41 +++ runtime-core/runtime-core.cmake | 25 ++ .../runtime-core.h | 113 ++------ runtime-core/utils/kphp-assert-core.h | 23 ++ .../utils/migration-php8.cpp | 9 +- .../utils/migration-php8.h | 2 - runtime/allocator.cpp | 2 +- runtime/allocator.h | 2 +- runtime/array_functions.h | 5 +- runtime/bcmath.cpp | 1 + runtime/bcmath.h | 2 +- runtime/common_template_instantiations.h | 2 +- runtime/confdata-functions.h | 2 +- runtime/confdata-global-manager.h | 6 +- runtime/confdata-keys.h | 3 +- runtime/context/runtime-context.cpp | 21 ++ runtime/context/runtime-context.h | 21 ++ runtime/context/runtime-core-allocator.cpp | 47 +++ runtime/context/runtime-core-context.cpp | 24 ++ runtime/ctype.h | 2 +- runtime/curl-async.h | 2 +- runtime/curl.cpp | 5 +- runtime/curl.h | 5 +- runtime/datetime/date_interval.h | 2 +- runtime/datetime/datetime.h | 6 +- runtime/datetime/datetime_functions.cpp | 3 +- runtime/datetime/datetime_functions.h | 2 +- runtime/datetime/datetime_immutable.h | 4 +- runtime/datetime/datetime_interface.h | 2 +- runtime/datetime/datetime_zone.h | 4 +- runtime/datetime/timelib_wrapper.cpp | 6 +- runtime/datetime/timelib_wrapper.h | 2 +- runtime/env.h | 2 +- runtime/exception.cpp | 9 +- runtime/exception.h | 6 +- runtime/exec.h | 2 +- runtime/ffi.h | 4 +- runtime/files.cpp | 5 +- runtime/files.h | 2 +- runtime/from-json-processor.h | 6 +- runtime/instance-cache.cpp | 4 +- runtime/instance-cache.h | 3 +- runtime/instance-copy-processor.h | 4 +- runtime/interface.cpp | 66 +++-- runtime/interface.h | 3 +- runtime/job-workers/client-functions.h | 2 +- runtime/job-workers/job-interface.h | 4 +- runtime/job-workers/processing-jobs.h | 2 +- runtime/job-workers/server-functions.h | 2 +- runtime/json-functions.cpp | 122 ++++---- runtime/json-functions.h | 33 +-- runtime/json-writer.cpp | 59 ++-- runtime/json-writer.h | 3 +- runtime/kphp-backtrace.h | 4 +- runtime/kphp_ml/kphp_ml.cpp | 2 +- runtime/kphp_ml/kphp_ml_catboost.cpp | 2 +- runtime/kphp_ml/kphp_ml_catboost.h | 2 +- runtime/kphp_ml/kphp_ml_init.h | 2 +- runtime/kphp_ml/kphp_ml_interface.h | 2 +- runtime/kphp_ml/kphp_ml_xgboost.cpp | 2 +- runtime/kphp_ml/kphp_ml_xgboost.h | 2 +- runtime/kphp_tracing.h | 4 +- runtime/kphp_tracing_binlog.cpp | 3 + runtime/kphp_tracing_binlog.h | 2 +- runtime/mail.h | 2 +- runtime/math_functions.h | 2 +- runtime/mbstring.h | 2 +- runtime/memcache.h | 6 +- runtime/memory_resource/memory_resource.cpp | 20 -- .../dealer.cpp | 2 +- .../dealer.h | 4 +- .../heap_resource.cpp | 2 +- .../heap_resource.h | 2 +- .../memory_resource_stats.cpp | 16 ++ .../memory_resource_stats.h | 10 + .../monotonic_runtime_buffer_resource.cpp} | 56 ++-- runtime/memory_usage.h | 46 ++- runtime/misc.cpp | 18 +- runtime/misc.h | 2 +- runtime/msgpack-serialization.h | 9 +- runtime/msgpack/adaptors.h | 2 +- runtime/msgpack/packer.cpp | 2 +- runtime/msgpack/unpacker.h | 2 +- runtime/mysql.h | 4 +- runtime/null_coalesce.h | 4 +- runtime/on_kphp_warning_callback.h | 2 +- runtime/openssl.cpp | 7 +- runtime/openssl.h | 2 +- runtime/pdo/abstract_pdo_driver.h | 2 +- runtime/pdo/abstract_pdo_statement.h | 2 +- runtime/pdo/mysql/mysql_pdo_driver.h | 2 +- .../pdo/mysql/mysql_pdo_emulated_statement.h | 2 +- runtime/pdo/pdo.h | 4 +- runtime/pdo/pdo_statement.h | 5 +- runtime/pdo/pgsql/pgsql_pdo_driver.h | 2 +- .../pgsql/pgsql_pdo_emulated_statement.cpp | 4 +- .../pdo/pgsql/pgsql_pdo_emulated_statement.h | 2 +- runtime/php-script-globals.h | 2 +- runtime/php_assert.cpp | 16 ++ runtime/php_assert.h | 24 +- runtime/profiler.h | 3 +- runtime/regexp.cpp | 35 ++- runtime/regexp.h | 14 +- runtime/resumable.h | 2 +- runtime/rpc.cpp | 9 +- runtime/rpc.h | 4 +- runtime/rpc_extra_info.h | 7 +- runtime/runtime.cmake | 21 +- runtime/serialize-functions.cpp | 50 ++-- runtime/serialize-functions.h | 16 +- runtime/spl/array_iterator.h | 4 +- runtime/storage.h | 2 +- runtime/streams.cpp | 1 + runtime/streams.h | 2 +- runtime/string-list.h | 3 +- runtime/string_buffer.cpp | 24 -- runtime/string_functions.cpp | 271 +++++++++--------- runtime/string_functions.h | 2 +- runtime/tcp.cpp | 1 + runtime/tcp.h | 2 +- runtime/tl/rpc_function.h | 4 +- runtime/tl/rpc_req_error.h | 2 +- runtime/tl/rpc_response.h | 4 +- runtime/tl/rpc_tl_query.h | 7 +- runtime/tl/tl_builtins.h | 2 +- runtime/tl/tl_func_base.h | 3 +- runtime/tl/tl_magics_decoding.h | 2 +- runtime/to-array-processor.h | 2 +- runtime/to-json-processor.h | 4 +- runtime/typed_rpc.h | 4 +- runtime/uber-h3.cpp | 2 + runtime/uber-h3.h | 2 +- runtime/udp.cpp | 1 + runtime/udp.h | 2 +- runtime/url.cpp | 44 +-- runtime/url.h | 8 +- runtime/vkext.cpp | 1 + runtime/vkext.h | 2 +- runtime/vkext_stats.h | 3 +- runtime/zlib.cpp | 13 +- runtime/zlib.h | 4 +- runtime/zstd.cpp | 2 +- runtime/zstd.h | 3 +- server/confdata-binlog-events.h | 2 +- server/confdata-binlog-replay.cpp | 1 + server/confdata-stats.cpp | 3 +- server/database-drivers/adaptor.h | 4 +- .../database-drivers/mysql/mysql-connector.h | 2 +- server/database-drivers/mysql/mysql-request.h | 2 +- server/database-drivers/mysql/mysql.h | 2 +- .../database-drivers/pgsql/pgsql-connector.h | 3 +- server/database-drivers/pgsql/pgsql-request.h | 2 +- server/database-drivers/pgsql/pgsql.h | 2 +- server/job-workers/job-message.h | 4 +- server/job-workers/job-stats.cpp | 2 +- server/job-workers/shared-memory-manager.h | 2 +- server/php-master.cpp | 3 +- server/php-runner.cpp | 1 + server/server-stats.h | 2 +- server/signal-handlers.cpp | 1 + server/statshouse/statshouse-manager.h | 2 +- tests/cpp/runtime/_runtime-tests-env.cpp | 1 + .../array-int-string-keys-collision-test.cpp | 2 +- tests/cpp/runtime/array-test.cpp | 2 +- tests/cpp/runtime/kphp-type-traits-test.cpp | 6 +- .../details/memory_chunk_list-test.cpp | 2 +- .../details/memory_chunk_tree-test.cpp | 4 +- .../memory_ordered_chunk_list-test.cpp | 4 +- .../memory_resource/details/test-helpers.h | 4 +- .../extra-memory-pool-test.cpp | 2 +- .../unsynchronized_pool_resource-test.cpp | 2 +- tests/cpp/runtime/msgpack-test.cpp | 2 +- .../cpp/runtime/number-string-comparison.cpp | 2 +- tests/cpp/runtime/string-test.cpp | 2 +- .../shared-memory-manager-test.cpp | 2 +- 217 files changed, 1117 insertions(+), 883 deletions(-) create mode 100644 runtime-core/allocator/script-allocator-managed.h rename runtime/class_instance_decl.inl => runtime-core/class-instance/class-instance-decl.inl (98%) rename runtime/class_instance.inl => runtime-core/class-instance/class-instance.inl (95%) rename runtime/refcountable_php_classes.h => runtime-core/class-instance/refcountable-php-classes.h (91%) rename {runtime => runtime-core/core-types}/comparison_operators.inl (98%) rename {runtime => runtime-core/core-types}/conversions_types.inl (98%) rename {runtime => runtime-core/core-types/decl}/array_decl.inl (99%) rename {runtime => runtime-core/core-types/decl}/array_iterator.h (99%) rename {runtime => runtime-core/core-types/decl}/declarations.h (100%) rename {runtime => runtime-core/core-types/decl}/mixed_decl.inl (99%) rename {runtime => runtime-core/core-types/decl}/optional.h (97%) rename {runtime => runtime-core/core-types/decl}/shape.h (100%) rename {runtime => runtime-core/core-types/decl}/string_buffer_decl.inl (86%) rename {runtime => runtime-core/core-types/decl}/string_decl.inl (99%) rename {runtime => runtime-core/core-types/definition}/array.inl (98%) rename {runtime => runtime-core/core-types/definition}/mixed.cpp (95%) rename {runtime => runtime-core/core-types/definition}/mixed.inl (99%) rename {runtime => runtime-core/core-types/definition}/string.cpp (90%) rename {runtime => runtime-core/core-types/definition}/string.inl (97%) create mode 100644 runtime-core/core-types/definition/string_buffer.cpp rename {runtime => runtime-core/core-types/definition}/string_buffer.inl (75%) rename {runtime => runtime-core/core-types/definition}/string_cache.cpp (95%) rename {runtime => runtime-core/core-types/definition}/string_cache.h (99%) rename {runtime => runtime-core/core-types}/kphp_type_traits.h (93%) rename {runtime => runtime-core}/include.h (84%) rename {runtime/memory_resource => runtime-core/memory-resource}/details/memory_chunk_list.h (94%) rename {runtime/memory_resource => runtime-core/memory-resource}/details/memory_chunk_tree.cpp (98%) rename {runtime/memory_resource => runtime-core/memory-resource}/details/memory_chunk_tree.h (96%) rename {runtime/memory_resource => runtime-core/memory-resource}/details/memory_ordered_chunk_list.cpp (97%) rename {runtime/memory_resource => runtime-core/memory-resource}/details/memory_ordered_chunk_list.h (94%) rename {runtime/memory_resource => runtime-core/memory-resource}/details/universal_reallocate.h (92%) rename {runtime/memory_resource => runtime-core/memory-resource}/extra-memory-pool.h (100%) rename {runtime/memory_resource => runtime-core/memory-resource}/memory_resource.h (93%) create mode 100644 runtime-core/memory-resource/monotonic_buffer_resource.cpp rename {runtime/memory_resource => runtime-core/memory-resource}/monotonic_buffer_resource.h (95%) rename {runtime/memory_resource => runtime-core/memory-resource}/resource_allocator.h (97%) rename {runtime/memory_resource => runtime-core/memory-resource}/unsynchronized_pool_resource.cpp (96%) rename {runtime/memory_resource => runtime-core/memory-resource}/unsynchronized_pool_resource.h (92%) create mode 100644 runtime-core/runtime-core-context.h create mode 100644 runtime-core/runtime-core.cmake rename runtime/kphp_core.h => runtime-core/runtime-core.h (91%) create mode 100644 runtime-core/utils/kphp-assert-core.h rename runtime/migration_php8.cpp => runtime-core/utils/migration-php8.cpp (59%) rename runtime/migration_php8.h => runtime-core/utils/migration-php8.h (89%) create mode 100644 runtime/context/runtime-context.cpp create mode 100644 runtime/context/runtime-context.h create mode 100644 runtime/context/runtime-core-allocator.cpp create mode 100644 runtime/context/runtime-core-context.cpp delete mode 100644 runtime/memory_resource/memory_resource.cpp rename runtime/{memory_resource => memory_resource_impl}/dealer.cpp (86%) rename runtime/{memory_resource => memory_resource_impl}/dealer.h (91%) rename runtime/{memory_resource => memory_resource_impl}/heap_resource.cpp (96%) rename runtime/{memory_resource => memory_resource_impl}/heap_resource.h (90%) create mode 100644 runtime/memory_resource_impl/memory_resource_stats.cpp create mode 100644 runtime/memory_resource_impl/memory_resource_stats.h rename runtime/{memory_resource/monotonic_buffer_resource.cpp => memory_resource_impl/monotonic_runtime_buffer_resource.cpp} (63%) delete mode 100644 runtime/string_buffer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 59e70955e5..280e882a48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ include(${COMMON_DIR}/common.cmake) include(${COMMON_DIR}/tl/tl.cmake) include(${COMMON_DIR}/unicode/unicode.cmake) +include(${BASE_DIR}/runtime-core/runtime-core.cmake) include(${BASE_DIR}/runtime/runtime.cmake) include(${BASE_DIR}/server/server.cmake) include(${BASE_DIR}/compiler/compiler.cmake) diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index 54444f053b..8c4f5e10e0 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -880,7 +880,7 @@ function vk_stats_hll_is_packed($hll ::: string) ::: bool; /** @kphp-extern-func-info cpp_template_call */ function vk_dot_product ($a ::: array, $b ::: array) ::: ^1[*] | ^2[*]; -/** defined in kphp_core.h **/ +/** defined in runtime-core.h **/ function likely ($x ::: bool) ::: bool; function unlikely ($x ::: bool) ::: bool; diff --git a/common/stats/provider.h b/common/stats/provider.h index d4b9e78ba5..d12ff3ec37 100644 --- a/common/stats/provider.h +++ b/common/stats/provider.h @@ -11,6 +11,7 @@ #include #include "common/stats/buffer.h" +#include "common/mixin/not_copyable.h" constexpr int am_get_memory_usage_self = 1; constexpr int am_get_memory_usage_overall = 2; diff --git a/docs/kphp-internals/developing-and-extending-kphp/contributing-to-kphp.md b/docs/kphp-internals/developing-and-extending-kphp/contributing-to-kphp.md index acd9b7bcf6..4f16175615 100644 --- a/docs/kphp-internals/developing-and-extending-kphp/contributing-to-kphp.md +++ b/docs/kphp-internals/developing-and-extending-kphp/contributing-to-kphp.md @@ -129,7 +129,7 @@ Then you think about type inferring. What type should using power operator lead On type inferring step, you introduce *recalc_power()*, call it as necessary, and implement given logic. Next, you need to tie codegeneration and C++ implementation together. -As you resulted in having 3 different inferrings, you need at least 3 C++ functions: say, you name them *int_power()*, *float_power()*, and *mixed_power()* and implement them somewhere in runtime — in *kphp_core.h* for example; the last one not only returns *mixed* but accepts *mixed* also, even though arguments could be inferred as clean types, they would be implicitly converted to *mixed* — it's easier to create a single function without lots overloads in this case. +As you resulted in having 3 different inferrings, you need at least 3 C++ functions: say, you name them *int_power()*, *float_power()*, and *mixed_power()* and implement them somewhere in runtime — in *runtime-core.h* for example; the last one not only returns *mixed* but accepts *mixed* also, even though arguments could be inferred as clean types, they would be implicitly converted to *mixed* — it's easier to create a single function without lots overloads in this case. On codegeneration of *op_pow*, you take the inferred result and output calling one of these functions. To support `**=`, you consider how `+=` and similar are made: "set operator" depends on "base operator". diff --git a/runtime-core/allocator/script-allocator-managed.h b/runtime-core/allocator/script-allocator-managed.h new file mode 100644 index 0000000000..eae0cb6125 --- /dev/null +++ b/runtime-core/allocator/script-allocator-managed.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +#include "runtime-core/runtime-core.h" + +class ScriptAllocatorManaged { +public: + static void *operator new(size_t size) noexcept { + return RuntimeAllocator::current().alloc_script_memory(size); + } + + static void *operator new(size_t, void *ptr) noexcept { + return ptr; + } + + static void operator delete(void *ptr, size_t size) noexcept { + RuntimeAllocator::current().free_script_memory(ptr, size); + } + + static void *operator new[](size_t count) = delete; + static void operator delete[](void *ptr, size_t sz) = delete; + static void operator delete[](void *ptr) = delete; + +protected: + ~ScriptAllocatorManaged() = default; +}; diff --git a/runtime/class_instance_decl.inl b/runtime-core/class-instance/class-instance-decl.inl similarity index 98% rename from runtime/class_instance_decl.inl rename to runtime-core/class-instance/class-instance-decl.inl index 1f32c75f1f..be0d3ed53d 100644 --- a/runtime/class_instance_decl.inl +++ b/runtime-core/class-instance/class-instance-decl.inl @@ -3,7 +3,7 @@ #include "common/smart_ptrs/intrusive_ptr.h" #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif // PHP classes produce the C++ structures of the form: diff --git a/runtime/class_instance.inl b/runtime-core/class-instance/class-instance.inl similarity index 95% rename from runtime/class_instance.inl rename to runtime-core/class-instance/class-instance.inl index bed1014eb1..a7f572ed5a 100644 --- a/runtime/class_instance.inl +++ b/runtime-core/class-instance/class-instance.inl @@ -1,7 +1,7 @@ #pragma once #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif template @@ -43,8 +43,7 @@ class_instance class_instance::alloc(Args &&... args) { template inline class_instance class_instance::empty_alloc() { static_assert(std::is_empty{}, "class T must be empty"); - static uint32_t obj; - obj++; + uint32_t obj = ++KphpCoreContext::current().empty_obj_count; new (&o) vk::intrusive_ptr(reinterpret_cast(obj)); return *this; } diff --git a/runtime/refcountable_php_classes.h b/runtime-core/class-instance/refcountable-php-classes.h similarity index 91% rename from runtime/refcountable_php_classes.h rename to runtime-core/class-instance/refcountable-php-classes.h index 01ada7358e..09b37fafdc 100644 --- a/runtime/refcountable_php_classes.h +++ b/runtime-core/class-instance/refcountable-php-classes.h @@ -7,9 +7,9 @@ #include "common/php-functions.h" -#include "runtime/allocator.h" +#include "runtime-core/allocator/script-allocator-managed.h" -class abstract_refcountable_php_interface : public ManagedThroughDlAllocator { +class abstract_refcountable_php_interface : public ScriptAllocatorManaged { public: abstract_refcountable_php_interface() __attribute__((always_inline)) = default; virtual ~abstract_refcountable_php_interface() noexcept __attribute__((always_inline)) = default; @@ -98,7 +98,7 @@ class refcountable_polymorphic_php_classes_virt<> : public virtual abstract_refc }; template -class refcountable_php_classes : public ManagedThroughDlAllocator { +class refcountable_php_classes : public ScriptAllocatorManaged { public: void add_ref() noexcept { if (refcnt < ExtraRefCnt::for_global_const) { @@ -118,7 +118,7 @@ class refcountable_php_classes : public ManagedThroughDlAllocator { if (refcnt == 0) { static_assert(!std::is_polymorphic{}, "Derived may not be polymorphic"); /** - * because of inheritance from ManagedThroughDlAllocator, which override operators new/delete + * because of inheritance from ScriptAllocatorManaged, which override operators new/delete * we should have vptr for passing proper sizeof of Derived class, but we don't want to increase size of every class * therefore we use static_cast here */ diff --git a/runtime/comparison_operators.inl b/runtime-core/core-types/comparison_operators.inl similarity index 98% rename from runtime/comparison_operators.inl rename to runtime-core/core-types/comparison_operators.inl index fe926a73e4..12e62d797e 100644 --- a/runtime/comparison_operators.inl +++ b/runtime-core/core-types/comparison_operators.inl @@ -1,9 +1,9 @@ #pragma once -#include "runtime/migration_php8.h" +#include "runtime-core/utils/migration-php8.h" #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif namespace impl_ { @@ -92,7 +92,7 @@ inline bool eq2_number_string_as_php8(T lhs, const string &rhs) { inline bool eq2(int64_t lhs, const string &rhs) { const auto php7_result = eq2(lhs, rhs.to_float()); - if (show_migration_php8_warning & MIGRATION_PHP8_STRING_COMPARISON_FLAG) { + if (KphpCoreContext::current().show_migration_php8_warning & MIGRATION_PHP8_STRING_COMPARISON_FLAG) { const auto php8_result = eq2_number_string_as_php8(lhs, rhs); if (php7_result == php8_result) { return php7_result; @@ -113,7 +113,7 @@ inline bool eq2(const string &lhs, int64_t rhs) { inline bool eq2(double lhs, const string &rhs) { const auto php7_result = lhs == rhs.to_float(); - if (show_migration_php8_warning & MIGRATION_PHP8_STRING_COMPARISON_FLAG) { + if (KphpCoreContext::current().show_migration_php8_warning & MIGRATION_PHP8_STRING_COMPARISON_FLAG) { const auto php8_result = eq2_number_string_as_php8(lhs, rhs); if (php7_result == php8_result) { return php7_result; diff --git a/runtime/conversions_types.inl b/runtime-core/core-types/conversions_types.inl similarity index 98% rename from runtime/conversions_types.inl rename to runtime-core/core-types/conversions_types.inl index 02ca00c551..a97f3e7296 100644 --- a/runtime/conversions_types.inl +++ b/runtime-core/core-types/conversions_types.inl @@ -1,7 +1,7 @@ #pragma once #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif template> diff --git a/runtime/array_decl.inl b/runtime-core/core-types/decl/array_decl.inl similarity index 99% rename from runtime/array_decl.inl rename to runtime-core/core-types/decl/array_decl.inl index 7071751743..007b536113 100644 --- a/runtime/array_decl.inl +++ b/runtime-core/core-types/decl/array_decl.inl @@ -4,11 +4,11 @@ #pragma once -#include "runtime/array_iterator.h" -#include "runtime/include.h" +#include "runtime-core/core-types/decl/array_iterator.h" +#include "runtime-core/include.h" #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif struct array_size { diff --git a/runtime/array_iterator.h b/runtime-core/core-types/decl/array_iterator.h similarity index 99% rename from runtime/array_iterator.h rename to runtime-core/core-types/decl/array_iterator.h index bc29803850..b03b59af9c 100644 --- a/runtime/array_iterator.h +++ b/runtime-core/core-types/decl/array_iterator.h @@ -7,7 +7,7 @@ #include "common/type_traits/list_of_types.h" #include "common/sanitizer.h" -#include "runtime/declarations.h" +#include "runtime-core/core-types/decl/declarations.h" template class array_iterator { diff --git a/runtime/declarations.h b/runtime-core/core-types/decl/declarations.h similarity index 100% rename from runtime/declarations.h rename to runtime-core/core-types/decl/declarations.h diff --git a/runtime/mixed_decl.inl b/runtime-core/core-types/decl/mixed_decl.inl similarity index 99% rename from runtime/mixed_decl.inl rename to runtime-core/core-types/decl/mixed_decl.inl index a3ae474615..cad5f864fc 100644 --- a/runtime/mixed_decl.inl +++ b/runtime-core/core-types/decl/mixed_decl.inl @@ -5,7 +5,7 @@ #pragma once #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif template diff --git a/runtime/optional.h b/runtime-core/core-types/decl/optional.h similarity index 97% rename from runtime/optional.h rename to runtime-core/core-types/decl/optional.h index 60d2b8d112..69dbde2a31 100644 --- a/runtime/optional.h +++ b/runtime-core/core-types/decl/optional.h @@ -10,8 +10,8 @@ #include "common/type_traits/is_constructible.h" #include "common/type_traits/list_of_types.h" -#include "runtime/php_assert.h" -#include "runtime/declarations.h" +#include "runtime-core/utils/kphp-assert-core.h" +#include "runtime-core/core-types/decl/declarations.h" template class Optional; diff --git a/runtime/shape.h b/runtime-core/core-types/decl/shape.h similarity index 100% rename from runtime/shape.h rename to runtime-core/core-types/decl/shape.h diff --git a/runtime/string_buffer_decl.inl b/runtime-core/core-types/decl/string_buffer_decl.inl similarity index 86% rename from runtime/string_buffer_decl.inl rename to runtime-core/core-types/decl/string_buffer_decl.inl index 34c52f7495..17ad43919f 100644 --- a/runtime/string_buffer_decl.inl +++ b/runtime-core/core-types/decl/string_buffer_decl.inl @@ -1,7 +1,7 @@ #pragma once #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif #define STRING_BUFFER_ERROR_FLAG_ON -1 @@ -9,8 +9,6 @@ #define STRING_BUFFER_ERROR_FLAG_FAILED 1 class string_buffer { - static string::size_type MIN_BUFFER_LEN; - static string::size_type MAX_BUFFER_LEN; char *buffer_end; char *buffer_begin; string::size_type buffer_len; @@ -21,7 +19,6 @@ class string_buffer { string_buffer &operator=(const string_buffer &other) = delete; public: - static int string_buffer_error_flag; explicit string_buffer(string::size_type buffer_len = 4000) noexcept; inline string_buffer &clean() noexcept; @@ -56,7 +53,7 @@ public: ~string_buffer() noexcept; - friend void init_string_buffer_lib(int max_length); + friend void init_string_buffer_lib(string::size_type min_length, string::size_type max_length); inline void debug_print() const; @@ -66,5 +63,8 @@ public: friend inline bool operator!=(const string_buffer &lhs, const string_buffer &rhs); }; -extern string_buffer static_SB; -extern string_buffer static_SB_spare; +struct string_buffer_lib_context { + string::size_type MIN_BUFFER_LEN; + string::size_type MAX_BUFFER_LEN; + int error_flag = 0; +}; diff --git a/runtime/string_decl.inl b/runtime-core/core-types/decl/string_decl.inl similarity index 99% rename from runtime/string_decl.inl rename to runtime-core/core-types/decl/string_decl.inl index 1d5132ab3e..b3d1c0eaa3 100644 --- a/runtime/string_decl.inl +++ b/runtime-core/core-types/decl/string_decl.inl @@ -1,7 +1,7 @@ #pragma once #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif using string_size_type = uint32_t; diff --git a/runtime/array.inl b/runtime-core/core-types/definition/array.inl similarity index 98% rename from runtime/array.inl rename to runtime-core/core-types/definition/array.inl index 367e79cad4..7db12ae63f 100644 --- a/runtime/array.inl +++ b/runtime-core/core-types/definition/array.inl @@ -9,7 +9,7 @@ #include "common/algorithms/fastmod.h" #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif array_size::array_size(int64_t int_size, bool is_vector) noexcept @@ -237,7 +237,7 @@ template typename array::array_inner *array::array_inner::create(int64_t new_int_size, bool is_vector) { const size_t mem_size = estimate_size(new_int_size, is_vector); if (is_vector) { - auto p = reinterpret_cast(dl::allocate(mem_size)); + auto p = reinterpret_cast(RuntimeAllocator::current().alloc_script_memory(mem_size)); p->is_vector_internal = true; p->ref_cnt = 0; p->max_key = -1; @@ -250,7 +250,7 @@ typename array::array_inner *array::array_inner::create(int64_t new_int_si return reinterpret_cast(static_cast(mem) + sizeof(array_inner_fields_for_map)); }; - array_inner *p = shift_pointer_to_array_inner(dl::allocate0(mem_size)); + array_inner *p = shift_pointer_to_array_inner(RuntimeAllocator::current().alloc0_script_memory(mem_size)); p->is_vector_internal = false; p->ref_cnt = 0; p->max_key = -1; @@ -275,7 +275,7 @@ void array::array_inner::dispose() { ((T *)entries)[i].~T(); } - dl::deallocate((void *)this, sizeof_vector(buf_size)); + RuntimeAllocator::current().free_script_memory((void *)this, sizeof_vector(buf_size)); return; } @@ -288,7 +288,7 @@ void array::array_inner::dispose() { php_assert(this != empty_array()); auto shifted_this = reinterpret_cast(this) - sizeof(array_inner_fields_for_map); - dl::deallocate(shifted_this, sizeof_map(buf_size)); + RuntimeAllocator::current().free_script_memory(shifted_this, sizeof_map(buf_size)); } } } @@ -716,7 +716,7 @@ void array::mutate_to_size(int64_t int_size) { php_critical_error ("max array size exceeded: int_size = %" PRIi64, int_size); } const auto new_int_buff_size = static_cast(int_size); - p = static_cast(dl::reallocate(p, p->sizeof_vector(new_int_buff_size), p->sizeof_vector(p->buf_size))); + p = static_cast(RuntimeAllocator::current().realloc_script_memory(p, p->sizeof_vector(new_int_buff_size), p->sizeof_vector(p->buf_size))); p->buf_size = new_int_buff_size; } @@ -1650,7 +1650,7 @@ array &array::operator+=(const array &other) { p = new_array; } else if (p->buf_size < size + 2) { uint32_t new_size = max(size + 2, p->buf_size * 2); - p = (array_inner *)dl::reallocate((void *)p, p->sizeof_vector(new_size), p->sizeof_vector(p->buf_size)); + p = (array_inner *)RuntimeAllocator::current().realloc_script_memory((void *)p, p->sizeof_vector(new_size), p->sizeof_vector(p->buf_size)); p->buf_size = new_size; } @@ -1894,7 +1894,7 @@ void array::sort(const T1 &compare, bool renumber) { mutate_if_map_shared(); } - array_bucket **arTmp = (array_bucket **)dl::allocate(n * sizeof(array_bucket * )); + array_bucket **arTmp = (array_bucket **)RuntimeAllocator::current().alloc_script_memory(n * sizeof(array_bucket * )); uint32_t i = 0; for (array_bucket *it = p->begin(); it != p->end(); it = p->next(it)) { arTmp[i++] = it; @@ -1916,7 +1916,7 @@ void array::sort(const T1 &compare, bool renumber) { arTmp[n - 1]->next = p->get_pointer(p->end()); p->end()->prev = p->get_pointer(arTmp[n - 1]); - dl::deallocate(arTmp, n * sizeof(array_bucket * )); + RuntimeAllocator::current().free_script_memory(arTmp, n * sizeof(array_bucket * )); } diff --git a/runtime/mixed.cpp b/runtime-core/core-types/definition/mixed.cpp similarity index 95% rename from runtime/mixed.cpp rename to runtime-core/core-types/definition/mixed.cpp index 27e4b01f5c..2ead9df246 100644 --- a/runtime/mixed.cpp +++ b/runtime-core/core-types/definition/mixed.cpp @@ -2,7 +2,7 @@ // Copyright (c) 2021 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" void mixed::destroy() noexcept { switch (get_type()) { diff --git a/runtime/mixed.inl b/runtime-core/core-types/definition/mixed.inl similarity index 99% rename from runtime/mixed.inl rename to runtime-core/core-types/definition/mixed.inl index c8f3742eac..9252a31fcf 100644 --- a/runtime/mixed.inl +++ b/runtime-core/core-types/definition/mixed.inl @@ -6,10 +6,10 @@ #include "common/algorithms/find.h" -#include "runtime/migration_php8.h" +#include "runtime-core/utils/migration-php8.h" #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif static_assert(vk::all_of_equal(sizeof(string), sizeof(double), sizeof(array)), "sizeof of array, string and double must be equal"); @@ -1760,7 +1760,7 @@ inline bool less_string_number_as_php8_impl(const string &lhs, T rhs) { template inline bool less_number_string_as_php8(bool php7_result, T lhs, const string &rhs) { - if (show_migration_php8_warning & MIGRATION_PHP8_STRING_COMPARISON_FLAG) { + if (KphpCoreContext::current().show_migration_php8_warning & MIGRATION_PHP8_STRING_COMPARISON_FLAG) { const auto php8_result = less_number_string_as_php8_impl(lhs, rhs); if (php7_result == php8_result) { return php7_result; @@ -1778,7 +1778,7 @@ inline bool less_number_string_as_php8(bool php7_result, T lhs, const string &rh template inline bool less_string_number_as_php8(bool php7_result, const string &lhs, T rhs) { - if (show_migration_php8_warning & MIGRATION_PHP8_STRING_COMPARISON_FLAG) { + if (KphpCoreContext::current().show_migration_php8_warning & MIGRATION_PHP8_STRING_COMPARISON_FLAG) { const auto php8_result = less_string_number_as_php8_impl(lhs, rhs); if (php7_result == php8_result) { return php7_result; diff --git a/runtime/string.cpp b/runtime-core/core-types/definition/string.cpp similarity index 90% rename from runtime/string.cpp rename to runtime-core/core-types/definition/string.cpp index c405722055..8c95a60314 100644 --- a/runtime/string.cpp +++ b/runtime-core/core-types/definition/string.cpp @@ -2,7 +2,7 @@ // Copyright (c) 2021 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" // Don't move this destructor to the headers, it spoils addr2line traces string::~string() noexcept { diff --git a/runtime/string.inl b/runtime-core/core-types/definition/string.inl similarity index 97% rename from runtime/string.inl rename to runtime-core/core-types/definition/string.inl index 0a577a0fd5..e8819e7865 100644 --- a/runtime/string.inl +++ b/runtime-core/core-types/definition/string.inl @@ -8,11 +8,11 @@ #include "common/algorithms/simd-int-to-string.h" -#include "runtime/string_cache.h" -#include "runtime/migration_php8.h" +#include "runtime-core/utils/migration-php8.h" +#include "runtime-core/core-types/definition/string_cache.h" #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif tmp_string::tmp_string(const char *data, string_size_type size) : data{data}, size{size} {} @@ -57,7 +57,7 @@ string::size_type string::string_inner::new_capacity(size_type requested_capacit string::string_inner *string::string_inner::create(size_type requested_capacity, size_type old_capacity) { size_type capacity = new_capacity(requested_capacity, old_capacity); size_type new_size = (size_type)(sizeof(string_inner) + (capacity + 1)); - string_inner *p = (string_inner *)dl::allocate(new_size); + string_inner *p = (string_inner *)RuntimeAllocator::current().alloc_script_memory(new_size); p->capacity = capacity; return p; } @@ -67,7 +67,7 @@ char *string::string_inner::reserve(size_type requested_capacity) { size_type old_size = (size_type)(sizeof(string_inner) + (capacity + 1)); size_type new_size = (size_type)(sizeof(string_inner) + (new_cap + 1)); - string_inner *p = (string_inner *)dl::reallocate((void *)this, new_size, old_size); + string_inner *p = (string_inner *)RuntimeAllocator::current().realloc_script_memory((void *)this, new_size, old_size); p->capacity = new_cap; return p->ref_data(); } @@ -83,7 +83,7 @@ void string::string_inner::dispose() { } void string::string_inner::destroy() { - dl::deallocate(this, get_memory_usage()); + RuntimeAllocator::current().free_script_memory(this, get_memory_usage()); } inline string::size_type string::string_inner::get_memory_usage() const { @@ -744,7 +744,7 @@ bool string::try_to_float_as_php7(double *val) const { bool string::try_to_float(double *val, bool php8_warning) const { const bool is_float_php7 = try_to_float_as_php7(val); - if ((show_migration_php8_warning & MIGRATION_PHP8_STRING_TO_FLOAT_FLAG) && php8_warning) { + if ((KphpCoreContext::current().show_migration_php8_warning & MIGRATION_PHP8_STRING_TO_FLOAT_FLAG) && php8_warning) { const bool is_float_php8 = try_to_float_as_php8(val); if (is_float_php7 != is_float_php8) { @@ -904,7 +904,7 @@ bool string::is_numeric_as_php7() const { bool string::is_numeric() const { const auto php7_result = is_numeric_as_php7(); - if (show_migration_php8_warning & MIGRATION_PHP8_STRING_TO_FLOAT_FLAG) { + if (KphpCoreContext::current().show_migration_php8_warning & MIGRATION_PHP8_STRING_TO_FLOAT_FLAG) { const bool php8_result = is_numeric_as_php8(); if (php7_result != php8_result) { diff --git a/runtime-core/core-types/definition/string_buffer.cpp b/runtime-core/core-types/definition/string_buffer.cpp new file mode 100644 index 0000000000..3d4c4c121a --- /dev/null +++ b/runtime-core/core-types/definition/string_buffer.cpp @@ -0,0 +1,15 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-core/runtime-core.h" + +string_buffer::string_buffer(string::size_type buffer_len) noexcept: + buffer_end(static_cast(RuntimeAllocator::current().alloc_global_memory(buffer_len))), + buffer_begin(buffer_end), + buffer_len(buffer_len) { +} + +string_buffer::~string_buffer() noexcept { + RuntimeAllocator::current().free_global_memory(buffer_begin, buffer_len); +} diff --git a/runtime/string_buffer.inl b/runtime-core/core-types/definition/string_buffer.inl similarity index 75% rename from runtime/string_buffer.inl rename to runtime-core/core-types/definition/string_buffer.inl index 567a8aa69f..744015dd22 100644 --- a/runtime/string_buffer.inl +++ b/runtime-core/core-types/definition/string_buffer.inl @@ -3,20 +3,21 @@ #include "common/algorithms/simd-int-to-string.h" #ifndef INCLUDED_FROM_KPHP_CORE - #error "this file must be included only from kphp_core.h" + #error "this file must be included only from runtime-core.h" #endif inline void string_buffer::resize(string::size_type new_buffer_len) noexcept { - if (new_buffer_len < MIN_BUFFER_LEN) { - new_buffer_len = MIN_BUFFER_LEN; + string_buffer_lib_context &sb_context = KphpCoreContext::current().sb_lib_context; + if (new_buffer_len < sb_context.MIN_BUFFER_LEN) { + new_buffer_len = sb_context.MIN_BUFFER_LEN; } - if (new_buffer_len >= MAX_BUFFER_LEN) { - if (buffer_len + 1 < MAX_BUFFER_LEN) { - new_buffer_len = MAX_BUFFER_LEN - 1; + if (new_buffer_len >= sb_context.MAX_BUFFER_LEN) { + if (buffer_len + 1 < sb_context.MAX_BUFFER_LEN) { + new_buffer_len = sb_context.MAX_BUFFER_LEN - 1; } else { - if (string_buffer_error_flag != STRING_BUFFER_ERROR_FLAG_OFF) { + if (sb_context.error_flag != STRING_BUFFER_ERROR_FLAG_OFF) { clean(); - string_buffer_error_flag = STRING_BUFFER_ERROR_FLAG_FAILED; + sb_context.error_flag = STRING_BUFFER_ERROR_FLAG_FAILED; return; } else { php_critical_error ("maximum buffer size exceeded. buffer_len = %u, new_buffer_len = %u", buffer_len, new_buffer_len); @@ -25,7 +26,7 @@ inline void string_buffer::resize(string::size_type new_buffer_len) noexcept { } string::size_type current_len = size(); - if(void *new_mem = dl::heap_reallocate(buffer_begin, new_buffer_len, buffer_len)) { + if(void *new_mem = RuntimeAllocator::current().realloc_global_memory(buffer_begin, new_buffer_len, buffer_len)) { buffer_begin = static_cast(new_mem); buffer_len = new_buffer_len; buffer_end = buffer_begin + current_len; @@ -34,7 +35,7 @@ inline void string_buffer::resize(string::size_type new_buffer_len) noexcept { inline void string_buffer::reserve_at_least(string::size_type need) noexcept { string::size_type new_buffer_len = need + size(); - while (unlikely (buffer_len < new_buffer_len && string_buffer_error_flag != STRING_BUFFER_ERROR_FLAG_FAILED)) { + while (unlikely (buffer_len < new_buffer_len && KphpCoreContext::current().sb_lib_context.error_flag != STRING_BUFFER_ERROR_FLAG_FAILED)) { resize(((new_buffer_len * 2 + 1 + 64) | 4095) - 64); } } @@ -71,7 +72,7 @@ string_buffer &operator<<(string_buffer &sb, const string &s) { string::size_type l = s.size(); sb.reserve_at_least(l); - if (unlikely (sb.string_buffer_error_flag == STRING_BUFFER_ERROR_FLAG_FAILED)) { + if (unlikely (KphpCoreContext::current().sb_lib_context.error_flag == STRING_BUFFER_ERROR_FLAG_FAILED)) { return sb; } @@ -141,7 +142,7 @@ bool string_buffer::set_pos(int64_t pos) { string_buffer &string_buffer::append(const char *str, size_t len) noexcept { reserve_at_least(static_cast(len)); - if (unlikely (string_buffer_error_flag == STRING_BUFFER_ERROR_FLAG_FAILED)) { + if (unlikely (KphpCoreContext::current().sb_lib_context.error_flag == STRING_BUFFER_ERROR_FLAG_FAILED)) { return *this; } memcpy(buffer_end, str, len); @@ -168,11 +169,13 @@ void string_buffer::reserve(int len) { reserve_at_least(len + 1); } -inline void init_string_buffer_lib(int max_length) { - string_buffer::MIN_BUFFER_LEN = 266175; - string_buffer::MAX_BUFFER_LEN = (1 << 24); +inline void init_string_buffer_lib(string::size_type min_length, string::size_type max_length) { + string_buffer_lib_context &sb_context = KphpCoreContext::current().sb_lib_context; + if (min_length > 0) { + sb_context.MIN_BUFFER_LEN = min_length; + } if (max_length > 0) { - string_buffer::MAX_BUFFER_LEN = max_length; + sb_context.MAX_BUFFER_LEN = max_length; } } diff --git a/runtime/string_cache.cpp b/runtime-core/core-types/definition/string_cache.cpp similarity index 95% rename from runtime/string_cache.cpp rename to runtime-core/core-types/definition/string_cache.cpp index ec823df121..19e62b700f 100644 --- a/runtime/string_cache.cpp +++ b/runtime-core/core-types/definition/string_cache.cpp @@ -2,7 +2,7 @@ // Copyright (c) 2020 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" constexpr auto string_cache::constexpr_make_large_ints() noexcept { return constexpr_make_ints(std::make_index_sequence{}); diff --git a/runtime/string_cache.h b/runtime-core/core-types/definition/string_cache.h similarity index 99% rename from runtime/string_cache.h rename to runtime-core/core-types/definition/string_cache.h index 645ca27833..66d5e289d9 100644 --- a/runtime/string_cache.h +++ b/runtime-core/core-types/definition/string_cache.h @@ -9,7 +9,7 @@ #include "common/php-functions.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" class string_cache { private: diff --git a/runtime/kphp_type_traits.h b/runtime-core/core-types/kphp_type_traits.h similarity index 93% rename from runtime/kphp_type_traits.h rename to runtime-core/core-types/kphp_type_traits.h index 93873c7e2f..0917cb0aea 100644 --- a/runtime/kphp_type_traits.h +++ b/runtime-core/core-types/kphp_type_traits.h @@ -8,9 +8,9 @@ #include "common/type_traits/list_of_types.h" -#include "runtime/declarations.h" -#include "runtime/optional.h" -#include "runtime/shape.h" +#include "runtime-core/core-types/decl/declarations.h" +#include "runtime-core/core-types/decl/optional.h" +#include "runtime-core/core-types/decl/shape.h" template struct is_array : std::false_type { diff --git a/runtime/include.h b/runtime-core/include.h similarity index 84% rename from runtime/include.h rename to runtime-core/include.h index 0195a50cc2..3c0070b6dd 100644 --- a/runtime/include.h +++ b/runtime-core/include.h @@ -11,10 +11,10 @@ #include "common/wrappers/likely.h" #include "common/php-functions.h" -#include "runtime/declarations.h" -#include "runtime/kphp_type_traits.h" -#include "runtime/optional.h" -#include "runtime/php_assert.h" +#include "runtime-core/core-types/decl/declarations.h" +#include "runtime-core/core-types/kphp_type_traits.h" +#include "runtime-core/core-types/decl/optional.h" +#include "runtime-core/utils/kphp-assert-core.h" #define COMMA , diff --git a/runtime/memory_resource/details/memory_chunk_list.h b/runtime-core/memory-resource/details/memory_chunk_list.h similarity index 94% rename from runtime/memory_resource/details/memory_chunk_list.h rename to runtime-core/memory-resource/details/memory_chunk_list.h index 2277318030..721499097a 100644 --- a/runtime/memory_resource/details/memory_chunk_list.h +++ b/runtime-core/memory-resource/details/memory_chunk_list.h @@ -6,7 +6,7 @@ #include -#include "runtime/memory_resource/memory_resource.h" +#include "runtime-core/memory-resource/memory_resource.h" namespace memory_resource { namespace details { diff --git a/runtime/memory_resource/details/memory_chunk_tree.cpp b/runtime-core/memory-resource/details/memory_chunk_tree.cpp similarity index 98% rename from runtime/memory_resource/details/memory_chunk_tree.cpp rename to runtime-core/memory-resource/details/memory_chunk_tree.cpp index b72d23bf77..e20383ee7a 100644 --- a/runtime/memory_resource/details/memory_chunk_tree.cpp +++ b/runtime-core/memory-resource/details/memory_chunk_tree.cpp @@ -2,13 +2,13 @@ // Copyright (c) 2020 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/memory_resource/details/memory_chunk_tree.h" +#include "memory_chunk_tree.h" #include #include -#include "runtime/memory_resource/details/memory_ordered_chunk_list.h" -#include "runtime/php_assert.h" +#include "runtime-core/memory-resource/details/memory_ordered_chunk_list.h" +#include "runtime-core/utils/kphp-assert-core.h" namespace memory_resource { namespace details { diff --git a/runtime/memory_resource/details/memory_chunk_tree.h b/runtime-core/memory-resource/details/memory_chunk_tree.h similarity index 96% rename from runtime/memory_resource/details/memory_chunk_tree.h rename to runtime-core/memory-resource/details/memory_chunk_tree.h index 53d60d3617..bc14e58108 100644 --- a/runtime/memory_resource/details/memory_chunk_tree.h +++ b/runtime-core/memory-resource/details/memory_chunk_tree.h @@ -6,7 +6,7 @@ #include "common/mixin/not_copyable.h" -#include "runtime/memory_resource/memory_resource.h" +#include "runtime-core/memory-resource/memory_resource.h" namespace memory_resource { namespace details { diff --git a/runtime/memory_resource/details/memory_ordered_chunk_list.cpp b/runtime-core/memory-resource/details/memory_ordered_chunk_list.cpp similarity index 97% rename from runtime/memory_resource/details/memory_ordered_chunk_list.cpp rename to runtime-core/memory-resource/details/memory_ordered_chunk_list.cpp index 96d9d3e60c..2458355120 100644 --- a/runtime/memory_resource/details/memory_ordered_chunk_list.cpp +++ b/runtime-core/memory-resource/details/memory_ordered_chunk_list.cpp @@ -2,7 +2,7 @@ // Copyright (c) 2020 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/memory_resource/details/memory_ordered_chunk_list.h" +#include "runtime-core/memory-resource/details/memory_ordered_chunk_list.h" #include #include diff --git a/runtime/memory_resource/details/memory_ordered_chunk_list.h b/runtime-core/memory-resource/details/memory_ordered_chunk_list.h similarity index 94% rename from runtime/memory_resource/details/memory_ordered_chunk_list.h rename to runtime-core/memory-resource/details/memory_ordered_chunk_list.h index c046d9fc04..da75fe1980 100644 --- a/runtime/memory_resource/details/memory_ordered_chunk_list.h +++ b/runtime-core/memory-resource/details/memory_ordered_chunk_list.h @@ -10,8 +10,8 @@ #include "common/mixin/not_copyable.h" -#include "runtime/memory_resource/memory_resource.h" -#include "runtime/php_assert.h" +#include "runtime-core/memory-resource/memory_resource.h" +#include "runtime-core/utils/kphp-assert-core.h" namespace memory_resource { namespace details { diff --git a/runtime/memory_resource/details/universal_reallocate.h b/runtime-core/memory-resource/details/universal_reallocate.h similarity index 92% rename from runtime/memory_resource/details/universal_reallocate.h rename to runtime-core/memory-resource/details/universal_reallocate.h index 977f016416..96cb13639f 100644 --- a/runtime/memory_resource/details/universal_reallocate.h +++ b/runtime-core/memory-resource/details/universal_reallocate.h @@ -6,7 +6,7 @@ #include -#include "runtime/memory_resource/memory_resource.h" +#include "runtime-core/memory-resource/memory_resource.h" namespace memory_resource { namespace details { diff --git a/runtime/memory_resource/extra-memory-pool.h b/runtime-core/memory-resource/extra-memory-pool.h similarity index 100% rename from runtime/memory_resource/extra-memory-pool.h rename to runtime-core/memory-resource/extra-memory-pool.h diff --git a/runtime/memory_resource/memory_resource.h b/runtime-core/memory-resource/memory_resource.h similarity index 93% rename from runtime/memory_resource/memory_resource.h rename to runtime-core/memory-resource/memory_resource.h index fe75638938..f17d65e4c8 100644 --- a/runtime/memory_resource/memory_resource.h +++ b/runtime-core/memory-resource/memory_resource.h @@ -9,8 +9,6 @@ #include #include -#include "common/stats/provider.h" - // #define DEBUG_MEMORY inline void memory_debug(const char *format, ...) __attribute__ ((format (printf, 1, 2))); @@ -48,8 +46,6 @@ class MemoryStats { size_t total_allocations{0}; // the total number of allocations size_t total_memory_allocated{0}; // the total amount of the memory allocated (doesn't take the freed memory into the account) - - void write_stats_to(stats_t *stats, const char *prefix) const noexcept; }; } // namespace memory_resource diff --git a/runtime-core/memory-resource/monotonic_buffer_resource.cpp b/runtime-core/memory-resource/monotonic_buffer_resource.cpp new file mode 100644 index 0000000000..bbc5370659 --- /dev/null +++ b/runtime-core/memory-resource/monotonic_buffer_resource.cpp @@ -0,0 +1,19 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-core/memory-resource/monotonic_buffer_resource.h" + +namespace memory_resource { + +void monotonic_buffer::init(void *buffer, size_t buffer_size) noexcept { + php_assert(buffer_size <= memory_buffer_limit()); + memory_begin_ = static_cast(buffer); + memory_current_ = memory_begin_; + memory_end_ = memory_begin_ + buffer_size; + + stats_ = MemoryStats{}; + stats_.memory_limit = buffer_size; +} + +} // namespace memory_resource diff --git a/runtime/memory_resource/monotonic_buffer_resource.h b/runtime-core/memory-resource/monotonic_buffer_resource.h similarity index 95% rename from runtime/memory_resource/monotonic_buffer_resource.h rename to runtime-core/memory-resource/monotonic_buffer_resource.h index 95679ec83d..7eb9cdbad3 100644 --- a/runtime/memory_resource/monotonic_buffer_resource.h +++ b/runtime-core/memory-resource/monotonic_buffer_resource.h @@ -10,9 +10,9 @@ #include "common/mixin/not_copyable.h" #include "common/wrappers/likely.h" -#include "runtime/memory_resource/memory_resource.h" -#include "runtime/memory_resource/details/universal_reallocate.h" -#include "runtime/php_assert.h" +#include "runtime-core/memory-resource/details/universal_reallocate.h" +#include "runtime-core/memory-resource/memory_resource.h" +#include "runtime-core/utils/kphp-assert-core.h" namespace memory_resource { @@ -46,8 +46,6 @@ class monotonic_buffer : vk::not_copyable { static_cast(mem) + size <= memory_current_; } - void critical_dump(void *mem, size_t size) const noexcept; - MemoryStats stats_; static_assert(sizeof(char) == 1, "sizeof char should be 1"); @@ -125,6 +123,7 @@ class monotonic_buffer_resource : protected monotonic_buffer { } protected: + void critical_dump(void *mem, size_t size) const noexcept; // this function should never be called from the nested/base context, // since all allocators have their own mem stats; // when signaling OOM, we want to see the root mem stats, not the diff --git a/runtime/memory_resource/resource_allocator.h b/runtime-core/memory-resource/resource_allocator.h similarity index 97% rename from runtime/memory_resource/resource_allocator.h rename to runtime-core/memory-resource/resource_allocator.h index aebc817cd4..d96cd1aee3 100644 --- a/runtime/memory_resource/resource_allocator.h +++ b/runtime-core/memory-resource/resource_allocator.h @@ -7,7 +7,7 @@ #include "common/wrappers/likely.h" -#include "runtime/php_assert.h" +#include "runtime-core/utils/kphp-assert-core.h" namespace memory_resource { diff --git a/runtime/memory_resource/unsynchronized_pool_resource.cpp b/runtime-core/memory-resource/unsynchronized_pool_resource.cpp similarity index 96% rename from runtime/memory_resource/unsynchronized_pool_resource.cpp rename to runtime-core/memory-resource/unsynchronized_pool_resource.cpp index cb98af6688..4c0a26481a 100644 --- a/runtime/memory_resource/unsynchronized_pool_resource.cpp +++ b/runtime-core/memory-resource/unsynchronized_pool_resource.cpp @@ -2,11 +2,11 @@ // Copyright (c) 2020 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/memory_resource/unsynchronized_pool_resource.h" +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" #include "common/wrappers/likely.h" -#include "runtime/memory_resource/details/memory_ordered_chunk_list.h" +#include "runtime-core/memory-resource/details/memory_ordered_chunk_list.h" namespace memory_resource { diff --git a/runtime/memory_resource/unsynchronized_pool_resource.h b/runtime-core/memory-resource/unsynchronized_pool_resource.h similarity index 92% rename from runtime/memory_resource/unsynchronized_pool_resource.h rename to runtime-core/memory-resource/unsynchronized_pool_resource.h index 3c47ec3c66..cc4fb2db3e 100644 --- a/runtime/memory_resource/unsynchronized_pool_resource.h +++ b/runtime-core/memory-resource/unsynchronized_pool_resource.h @@ -6,13 +6,13 @@ #include -#include "runtime/memory_resource/details/memory_chunk_list.h" -#include "runtime/memory_resource/details/memory_chunk_tree.h" -#include "runtime/memory_resource/details/universal_reallocate.h" -#include "runtime/memory_resource/extra-memory-pool.h" -#include "runtime/memory_resource/monotonic_buffer_resource.h" -#include "runtime/memory_resource/resource_allocator.h" -#include "runtime/php_assert.h" +#include "runtime-core/memory-resource/extra-memory-pool.h" +#include "runtime-core/memory-resource/details/memory_chunk_list.h" +#include "runtime-core/memory-resource/details/memory_chunk_tree.h" +#include "runtime-core/memory-resource/details/universal_reallocate.h" +#include "runtime-core/memory-resource/monotonic_buffer_resource.h" +#include "runtime-core/memory-resource/resource_allocator.h" +#include "runtime-core/utils/kphp-assert-core.h" namespace memory_resource { diff --git a/runtime-core/runtime-core-context.h b/runtime-core/runtime-core-context.h new file mode 100644 index 0000000000..b8ef96c820 --- /dev/null +++ b/runtime-core/runtime-core-context.h @@ -0,0 +1,41 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" + +#ifndef INCLUDED_FROM_KPHP_CORE + #error "this file must be included only from runtime-core.h" +#endif + +struct RuntimeAllocator { + static RuntimeAllocator& current() noexcept; + + void init(void * buffer, size_t script_mem_size, size_t oom_handling_mem_size); + void free(); + + void * alloc_script_memory(size_t size) noexcept; + void * alloc0_script_memory(size_t size) noexcept; + void * realloc_script_memory(void *mem, size_t new_size, size_t old_size) noexcept; + void free_script_memory(void *mem, size_t size) noexcept; + + void * alloc_global_memory(size_t size) noexcept; + void * realloc_global_memory(void *mem, size_t new_size, size_t old_size) noexcept; + void free_global_memory(void *mem, size_t size) noexcept; + + memory_resource::unsynchronized_pool_resource memory_resource; +}; + +struct KphpCoreContext { + + static KphpCoreContext& current() noexcept; + void init(); + void free(); + + int show_migration_php8_warning = 0; + uint32_t empty_obj_count = 0; + string_buffer_lib_context sb_lib_context; +}; diff --git a/runtime-core/runtime-core.cmake b/runtime-core/runtime-core.cmake new file mode 100644 index 0000000000..be50be3f42 --- /dev/null +++ b/runtime-core/runtime-core.cmake @@ -0,0 +1,25 @@ +prepend(KPHP_CORE_RUNTIME_UTILS ${BASE_DIR}/runtime-core/utils/ + migration-php8.cpp +) + +prepend(KPHP_CORE_TYPES ${BASE_DIR}/runtime-core/core-types/definition/ + mixed.cpp + string.cpp + string_buffer.cpp + string_cache.cpp +) + +prepend(KPHP_CORE_MEMORY_RESOURCE ${BASE_DIR}/runtime-core/memory-resource/ + details/memory_chunk_tree.cpp + details/memory_ordered_chunk_list.cpp + monotonic_buffer_resource.cpp + unsynchronized_pool_resource.cpp +) + +set(KPHP_CORE_SRC + ${KPHP_CORE_RUNTIME_UTILS} + ${KPHP_CORE_TYPES} + ${KPHP_CORE_MEMORY_RESOURCE} +) + +vk_add_library(runtime-core OBJECT ${KPHP_CORE_SRC}) diff --git a/runtime/kphp_core.h b/runtime-core/runtime-core.h similarity index 91% rename from runtime/kphp_core.h rename to runtime-core/runtime-core.h index dff0300cce..cc262a2f09 100644 --- a/runtime/kphp_core.h +++ b/runtime-core/runtime-core.h @@ -4,35 +4,43 @@ #pragma once +#include +#include +#include +#include +#include +#include #include +#include #include #include "common/algorithms/find.h" #include "common/sanitizer.h" #include "common/type_traits/list_of_types.h" -#include "runtime/allocator.h" -#include "runtime/include.h" -#include "runtime/kphp_type_traits.h" -#include "runtime/shape.h" +#include "runtime-core/include.h" +#include "runtime-core/core-types/kphp_type_traits.h" +#include "runtime-core/core-types/decl/shape.h" // order of includes below matters, be careful #define INCLUDED_FROM_KPHP_CORE -#include "string_decl.inl" -#include "array_decl.inl" -#include "class_instance_decl.inl" -#include "mixed_decl.inl" -#include "string_buffer_decl.inl" +#include "runtime-core/core-types/decl/string_decl.inl" +#include "runtime-core/core-types/decl/array_decl.inl" +#include "runtime-core/class-instance/class-instance-decl.inl" +#include "runtime-core/core-types/decl/mixed_decl.inl" +#include "runtime-core/core-types/decl/string_buffer_decl.inl" -#include "string.inl" -#include "array.inl" -#include "class_instance.inl" -#include "mixed.inl" -#include "string_buffer.inl" -#include "conversions_types.inl" -#include "comparison_operators.inl" +#include "runtime-core/runtime-core-context.h" + +#include "runtime-core/core-types/definition/string.inl" +#include "runtime-core/core-types/definition/array.inl" +#include "runtime-core/class-instance/class-instance.inl" +#include "runtime-core/core-types/definition/mixed.inl" +#include "runtime-core/core-types/definition/string_buffer.inl" +#include "runtime-core/core-types/conversions_types.inl" +#include "runtime-core/core-types/comparison_operators.inl" #undef INCLUDED_FROM_KPHP_CORE @@ -460,27 +468,11 @@ constexpr int32_t E_ALL = 32767; inline mixed f$error_get_last(); -inline int64_t f$error_reporting(int64_t level); - -inline int64_t f$error_reporting(); - inline void f$warning(const string &message); #define f$critical_error(message) \ php_critical_error("%s", message.c_str()); -inline int64_t f$memory_get_static_usage(); - -inline int64_t f$memory_get_peak_usage(bool real_usage = false); - -inline int64_t f$memory_get_usage(bool real_usage = false); - -inline int64_t f$memory_get_total_usage(); - -inline array f$memory_get_detailed_stats(); - -inline std::tuple f$memory_get_allocations(); - template inline int64_t f$get_reference_counter(const array &v); @@ -1324,67 +1316,10 @@ mixed f$error_get_last() { return {}; } -int64_t f$error_reporting(int64_t level) { - int32_t prev = php_warning_level; - if ((level & E_ALL) == E_ALL) { - php_warning_level = 3; - } - if (0 <= level && level <= 3) { - php_warning_level = std::max(php_warning_minimum_level, static_cast(level)); - } - return prev; -} - -int64_t f$error_reporting() { - return php_warning_level; -} - void f$warning(const string &message) { php_warning("%s", message.c_str()); } -int64_t f$memory_get_static_usage() { - return static_cast(dl::get_heap_memory_used()); -} - -int64_t f$memory_get_peak_usage(bool real_usage) { - if (real_usage) { - return static_cast(dl::get_script_memory_stats().max_real_memory_used); - } else { - return static_cast(dl::get_script_memory_stats().max_memory_used); - } -} - -int64_t f$memory_get_usage(bool real_usage __attribute__((unused))) { - return static_cast(dl::get_script_memory_stats().memory_used); -} - -int64_t f$memory_get_total_usage() { - return static_cast(dl::get_script_memory_stats().real_memory_used); -} - -array f$memory_get_detailed_stats() { - const auto &stats = dl::get_script_memory_stats(); - return array( - { - std::make_pair(string{"memory_limit"}, static_cast(stats.memory_limit)), - std::make_pair(string{"real_memory_used"}, static_cast(stats.real_memory_used)), - std::make_pair(string{"memory_used"}, static_cast(stats.memory_used)), - std::make_pair(string{"max_real_memory_used"}, static_cast(stats.max_real_memory_used)), - std::make_pair(string{"max_memory_used"}, static_cast(stats.max_memory_used)), - std::make_pair(string{"defragmentation_calls"}, static_cast(stats.defragmentation_calls)), - std::make_pair(string{"huge_memory_pieces"}, static_cast(stats.huge_memory_pieces)), - std::make_pair(string{"small_memory_pieces"}, static_cast(stats.small_memory_pieces)), - std::make_pair(string{"heap_memory_used"}, static_cast(dl::get_heap_memory_used())) - }); -} - -std::tuple f$memory_get_allocations() { - const auto &stats = dl::get_script_memory_stats(); - return {stats.total_allocations, stats.total_memory_allocated}; -} - - template int64_t f$get_reference_counter(const array &v) { return v.get_reference_counter(); diff --git a/runtime-core/utils/kphp-assert-core.h b/runtime-core/utils/kphp-assert-core.h new file mode 100644 index 0000000000..44024d5508 --- /dev/null +++ b/runtime-core/utils/kphp-assert-core.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include "common/wrappers/likely.h" + +void php_notice(char const *message, ...) __attribute__ ((format (printf, 1, 2))); +void php_warning(char const *message, ...) __attribute__ ((format (printf, 1, 2))); +void php_error(char const *message, ...) __attribute__ ((format (printf, 1, 2))); + +[[noreturn]] void php_assert__(const char *msg, const char *file, int line); +[[noreturn]] void raise_php_assert_signal__(); + +#define php_assert(EX) do { \ + if (unlikely(!(EX))) { \ + php_assert__ (#EX, __FILE__, __LINE__); \ + } \ +} while(0) + +#define php_critical_error(format, ...) do { \ + php_error ("Critical error \"" format "\" in file %s on line %d", ##__VA_ARGS__, __FILE__, __LINE__); \ + raise_php_assert_signal__(); \ +} while(0) diff --git a/runtime/migration_php8.cpp b/runtime-core/utils/migration-php8.cpp similarity index 59% rename from runtime/migration_php8.cpp rename to runtime-core/utils/migration-php8.cpp index 40553c7e58..c003c519a9 100644 --- a/runtime/migration_php8.cpp +++ b/runtime-core/utils/migration-php8.cpp @@ -2,16 +2,15 @@ // Copyright (c) 2021 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "migration_php8.h" - -int show_migration_php8_warning = 0; +#include "runtime-core/utils/migration-php8.h" +#include "runtime-core/runtime-core.h" void f$set_migration_php8_warning(int mask) { - show_migration_php8_warning = mask; + KphpCoreContext::current().show_migration_php8_warning = mask; } static void reset_migration_php8_global_vars() { - show_migration_php8_warning = 0; + KphpCoreContext::current().show_migration_php8_warning = 0; } void free_migration_php8() { diff --git a/runtime/migration_php8.h b/runtime-core/utils/migration-php8.h similarity index 89% rename from runtime/migration_php8.h rename to runtime-core/utils/migration-php8.h index eddedc8caf..edcca7471f 100644 --- a/runtime/migration_php8.h +++ b/runtime-core/utils/migration-php8.h @@ -7,8 +7,6 @@ constexpr int MIGRATION_PHP8_STRING_COMPARISON_FLAG = 1 << 0; constexpr int MIGRATION_PHP8_STRING_TO_FLOAT_FLAG = 1 << 1; -extern int show_migration_php8_warning; - void f$set_migration_php8_warning(int mask); void free_migration_php8(); diff --git a/runtime/allocator.cpp b/runtime/allocator.cpp index e95744a8aa..f4682f862f 100644 --- a/runtime/allocator.cpp +++ b/runtime/allocator.cpp @@ -17,7 +17,7 @@ #include "runtime/critical_section.h" #include "runtime/kphp-backtrace.h" -#include "runtime/memory_resource/dealer.h" +#include "runtime/memory_resource_impl//dealer.h" #include "runtime/php_assert.h" #include "server/server-log.h" diff --git a/runtime/allocator.h b/runtime/allocator.h index 57345f7e9d..2ccf785f05 100644 --- a/runtime/allocator.h +++ b/runtime/allocator.h @@ -9,7 +9,7 @@ #include #include "common/containers/final_action.h" -#include "runtime/memory_resource/memory_resource.h" +#include "runtime-core/memory-resource/memory_resource.h" namespace memory_resource { class unsynchronized_pool_resource; diff --git a/runtime/array_functions.h b/runtime/array_functions.h index 40568b13a3..11b5b40aff 100644 --- a/runtime/array_functions.h +++ b/runtime/array_functions.h @@ -10,7 +10,8 @@ #include "common/type_traits/function_traits.h" #include "common/vector-product.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" +#include "runtime/context/runtime-context.h" #include "runtime/math_functions.h" #include "runtime/string_functions.h" @@ -323,7 +324,7 @@ string f$implode(const string &s, const array &a) { // fallback to the generic iterator + string_buffer solution - string_buffer &SB = static_SB; + string_buffer &SB = kphp_runtime_context.static_SB; SB.clean(); auto it = a.begin(), it_end = a.end(); diff --git a/runtime/bcmath.cpp b/runtime/bcmath.cpp index e807e46b3d..7560a60840 100644 --- a/runtime/bcmath.cpp +++ b/runtime/bcmath.cpp @@ -3,6 +3,7 @@ // Distributed under the GPL v3 License, see LICENSE.notice.txt #include "runtime/bcmath.h" +#include "runtime/allocator.h" namespace { diff --git a/runtime/bcmath.h b/runtime/bcmath.h index d98da5bbb7..53836e8a4e 100644 --- a/runtime/bcmath.h +++ b/runtime/bcmath.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" void f$bcscale(int64_t scale); diff --git a/runtime/common_template_instantiations.h b/runtime/common_template_instantiations.h index 6c520084f4..94582cbf82 100644 --- a/runtime/common_template_instantiations.h +++ b/runtime/common_template_instantiations.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" // Use explicit template instantiation to make result binary smaller and force common instantiations to be compiled with -O3 // see https://en.cppreference.com/w/cpp/language/class_template diff --git a/runtime/confdata-functions.h b/runtime/confdata-functions.h index 402ad7e480..56b4e7104e 100644 --- a/runtime/confdata-functions.h +++ b/runtime/confdata-functions.h @@ -3,7 +3,7 @@ // Distributed under the GPL v3 License, see LICENSE.notice.txt #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" void init_confdata_functions_lib(); void free_confdata_functions_lib(); diff --git a/runtime/confdata-global-manager.h b/runtime/confdata-global-manager.h index 57d223a275..12e8920eee 100644 --- a/runtime/confdata-global-manager.h +++ b/runtime/confdata-global-manager.h @@ -9,11 +9,11 @@ #include "common/mixin/not_copyable.h" #include "common/wrappers/string_view.h" +#include "runtime-core/runtime-core.h" +#include "runtime-core/memory-resource/resource_allocator.h" +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" #include "runtime/confdata-keys.h" #include "runtime/inter-process-resource.h" -#include "runtime/kphp_core.h" -#include "runtime/memory_resource/resource_allocator.h" -#include "runtime/memory_resource/unsynchronized_pool_resource.h" using confdata_sample_storage = memory_resource::stl::map; diff --git a/runtime/confdata-keys.h b/runtime/confdata-keys.h index 10a5720b4c..8cbd9150e4 100644 --- a/runtime/confdata-keys.h +++ b/runtime/confdata-keys.h @@ -9,12 +9,13 @@ #include #include #include +#include #include "common/mixin/not_copyable.h" #include "common/wrappers/iterator_range.h" #include "common/wrappers/string_view.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" enum class ConfdataFirstKeyType { simple_key, diff --git a/runtime/context/runtime-context.cpp b/runtime/context/runtime-context.cpp new file mode 100644 index 0000000000..bc4bc48ffb --- /dev/null +++ b/runtime/context/runtime-context.cpp @@ -0,0 +1,21 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-context.h" + +#include "common/kprintf.h" +#include "runtime/allocator.h" + +KphpRuntimeContext kphp_runtime_context; +RuntimeAllocator runtime_allocator; + +void KphpRuntimeContext::init(void *mem, size_t script_mem_size, size_t oom_handling_mem_size) { + runtime_allocator.init(mem, script_mem_size, oom_handling_mem_size); + KphpCoreContext::init(); +} + +void KphpRuntimeContext::free() { + KphpCoreContext::free(); + runtime_allocator.free(); +} diff --git a/runtime/context/runtime-context.h b/runtime/context/runtime-context.h new file mode 100644 index 0000000000..68e7f110c0 --- /dev/null +++ b/runtime/context/runtime-context.h @@ -0,0 +1,21 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-core/runtime-core.h" + +#include "common/smart_ptrs/singleton.h" + +struct KphpRuntimeContext : KphpCoreContext { + + void init(void *mem, size_t script_mem_size, size_t oom_handling_mem_size); + void free(); + + string_buffer static_SB; + string_buffer static_SB_spare; +}; + +extern KphpRuntimeContext kphp_runtime_context; +extern RuntimeAllocator runtime_allocator; diff --git a/runtime/context/runtime-core-allocator.cpp b/runtime/context/runtime-core-allocator.cpp new file mode 100644 index 0000000000..143f02da17 --- /dev/null +++ b/runtime/context/runtime-core-allocator.cpp @@ -0,0 +1,47 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-core/runtime-core.h" +#include "runtime/allocator.h" +#include "runtime/context/runtime-context.h" + +void RuntimeAllocator::init(void *buffer, size_t script_mem_size, size_t oom_handling_mem_size) { + dl::init_script_allocator(buffer, script_mem_size, oom_handling_mem_size); +} + +void RuntimeAllocator::free() { + dl::free_script_allocator(); +} + +RuntimeAllocator &RuntimeAllocator::current() noexcept { + return runtime_allocator; +} + +void *RuntimeAllocator::alloc_script_memory(size_t size) noexcept { + return dl::allocate(size); +} + +void *RuntimeAllocator::alloc0_script_memory(size_t size) noexcept { + return dl::allocate0(size); +} + +void *RuntimeAllocator::realloc_script_memory(void *mem, size_t new_size, size_t old_size) noexcept { + return dl::reallocate(mem, new_size, old_size); +} + +void RuntimeAllocator::free_script_memory(void *mem, size_t size) noexcept { + dl::deallocate(mem, size); +} + +void *RuntimeAllocator::alloc_global_memory(size_t size) noexcept { + return dl::heap_allocate(size); +} + +void *RuntimeAllocator::realloc_global_memory(void *mem, size_t new_size, size_t old_size) noexcept { + return dl::heap_reallocate(mem, new_size, old_size); +} + +void RuntimeAllocator::free_global_memory(void *mem, size_t size) noexcept { + dl::heap_deallocate(mem, size); +} diff --git a/runtime/context/runtime-core-context.cpp b/runtime/context/runtime-core-context.cpp new file mode 100644 index 0000000000..b14f1a0f4f --- /dev/null +++ b/runtime/context/runtime-core-context.cpp @@ -0,0 +1,24 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "common/smart_ptrs/singleton.h" +#include "runtime/context/runtime-context.h" +#include "server/php-engine-vars.h" +#include "runtime/allocator.h" + +KphpCoreContext &KphpCoreContext::current() noexcept { + return kphp_runtime_context; +} + +void KphpCoreContext::init() { + if (static_buffer_length_limit < 0) { + init_string_buffer_lib(266175, (1 << 24)); + } else { + init_string_buffer_lib(266175, static_buffer_length_limit); + } +} + +void KphpCoreContext::free() { + free_migration_php8(); +} diff --git a/runtime/ctype.h b/runtime/ctype.h index 180a0a048b..a029767bfd 100644 --- a/runtime/ctype.h +++ b/runtime/ctype.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" bool f$ctype_alnum(const mixed &text) noexcept; bool f$ctype_alpha(const mixed &text) noexcept; diff --git a/runtime/curl-async.h b/runtime/curl-async.h index 4060ca37c3..af1fc37ef9 100644 --- a/runtime/curl-async.h +++ b/runtime/curl-async.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/curl.h" diff --git a/runtime/curl.cpp b/runtime/curl.cpp index 5d035e4a2e..c22ef194e6 100644 --- a/runtime/curl.cpp +++ b/runtime/curl.cpp @@ -10,6 +10,7 @@ #include #include +#include "runtime/context/runtime-context.h" #include "runtime/critical_section.h" #include "runtime/interface.h" #include "runtime/kphp_tracing.h" @@ -237,9 +238,9 @@ size_t curl_write(char *data, size_t size, size_t nmemb, void *userdata) { if (easy_context->return_transfer) { return easy_context->received_data.push_string(data, length) ? length : 0; } - string_buffer::string_buffer_error_flag = STRING_BUFFER_ERROR_FLAG_ON; + kphp_runtime_context.sb_lib_context.error_flag = STRING_BUFFER_ERROR_FLAG_ON; print(data, length); - return std::exchange(string_buffer::string_buffer_error_flag, STRING_BUFFER_ERROR_FLAG_OFF) == STRING_BUFFER_ERROR_FLAG_FAILED ? 0 : length; + return std::exchange(kphp_runtime_context.sb_lib_context.error_flag, STRING_BUFFER_ERROR_FLAG_OFF) == STRING_BUFFER_ERROR_FLAG_FAILED ? 0 : length; } // this is a callback called from curl_easy_perform diff --git a/runtime/curl.h b/runtime/curl.h index 77c7ca26ac..516d8b8724 100644 --- a/runtime/curl.h +++ b/runtime/curl.h @@ -5,8 +5,9 @@ #pragma once #include "common/smart_ptrs/singleton.h" - -#include "runtime/kphp_core.h" +#include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" +#include "runtime/allocator.h" using curl_easy = int64_t; diff --git a/runtime/datetime/date_interval.h b/runtime/datetime/date_interval.h index 8116962db0..c6e753dbbc 100644 --- a/runtime/datetime/date_interval.h +++ b/runtime/datetime/date_interval.h @@ -4,9 +4,9 @@ #pragma once +#include "runtime-core/class-instance/refcountable-php-classes.h" #include "runtime/datetime/timelib_wrapper.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/refcountable_php_classes.h" struct C$DateInterval: public refcountable_php_classes, private DummyVisitorMethods { using DummyVisitorMethods::accept; diff --git a/runtime/datetime/datetime.h b/runtime/datetime/datetime.h index a83a138ac5..f5b8edbdde 100644 --- a/runtime/datetime/datetime.h +++ b/runtime/datetime/datetime.h @@ -4,11 +4,11 @@ #pragma once -#include "runtime/datetime/datetime_zone.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/datetime/datetime_interface.h" +#include "runtime/datetime/datetime_zone.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/kphp_core.h" -#include "runtime/refcountable_php_classes.h" struct C$DateInterval; struct C$DateTimeImmutable; diff --git a/runtime/datetime/datetime_functions.cpp b/runtime/datetime/datetime_functions.cpp index de744bae17..0bd9f0eede 100644 --- a/runtime/datetime/datetime_functions.cpp +++ b/runtime/datetime/datetime_functions.cpp @@ -9,6 +9,7 @@ #include #include +#include "runtime/context/runtime-context.h" #include "runtime/critical_section.h" #include "runtime/datetime/timelib_wrapper.h" #include "runtime/string_functions.h" @@ -116,7 +117,7 @@ void iso_week_number(int y, int doy, int weekday, int &iw, int &iy) { static string date(const string &format, const tm &t, int64_t timestamp, bool local) { - string_buffer &SB = static_SB_spare; + string_buffer &SB = kphp_runtime_context.static_SB_spare; int year = t.tm_year + 1900; int month = t.tm_mon + 1; diff --git a/runtime/datetime/datetime_functions.h b/runtime/datetime/datetime_functions.h index 8b307e8c22..637e5b4b85 100644 --- a/runtime/datetime/datetime_functions.h +++ b/runtime/datetime/datetime_functions.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" bool f$checkdate(int64_t month, int64_t day, int64_t year); diff --git a/runtime/datetime/datetime_immutable.h b/runtime/datetime/datetime_immutable.h index 2104dc146a..fb4c877a52 100644 --- a/runtime/datetime/datetime_immutable.h +++ b/runtime/datetime/datetime_immutable.h @@ -4,11 +4,11 @@ #pragma once +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/datetime/datetime_interface.h" #include "runtime/datetime/datetime_zone.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/kphp_core.h" -#include "runtime/refcountable_php_classes.h" struct C$DateInterval; struct C$DateTime; diff --git a/runtime/datetime/datetime_interface.h b/runtime/datetime/datetime_interface.h index 158ba9624c..5d9222631a 100644 --- a/runtime/datetime/datetime_interface.h +++ b/runtime/datetime/datetime_interface.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/refcountable_php_classes.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" #include "runtime/datetime/timelib_wrapper.h" diff --git a/runtime/datetime/datetime_zone.h b/runtime/datetime/datetime_zone.h index 48a18d6496..fdad631343 100644 --- a/runtime/datetime/datetime_zone.h +++ b/runtime/datetime/datetime_zone.h @@ -4,9 +4,9 @@ #pragma once +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/kphp_core.h" -#include "runtime/refcountable_php_classes.h" struct C$DateTimeZone : public refcountable_php_classes, private DummyVisitorMethods { using DummyVisitorMethods::accept; diff --git a/runtime/datetime/timelib_wrapper.cpp b/runtime/datetime/timelib_wrapper.cpp index a569966ab6..f7661a8051 100644 --- a/runtime/datetime/timelib_wrapper.cpp +++ b/runtime/datetime/timelib_wrapper.cpp @@ -8,6 +8,8 @@ #include "common/containers/final_action.h" #include "common/smart_ptrs/singleton.h" +#include "runtime/allocator.h" +#include "runtime/context/runtime-context.h" #include "server/php-engine-vars.h" #include "server/php-runner.h" @@ -431,7 +433,7 @@ string php_timelib_date_format(const string &format, timelib_time *t, bool local auto script_guard = make_malloc_replacement_with_script_allocator(); - string_buffer &SB = static_SB_spare; + string_buffer &SB = kphp_runtime_context.static_SB_spare; SB.clean(); timelib_time_offset *offset = localtime ? create_time_offset(t, script_guard) : nullptr; @@ -859,7 +861,7 @@ string php_timelib_date_interval_format(const string &format, timelib_rel_time * return {}; } - string_buffer &SB = static_SB_spare; + string_buffer &SB = kphp_runtime_context.static_SB_spare; SB.clean(); // php implementation has 33 bytes buffer capacity, we have 128 bytes as well as php_timelib_date_format() diff --git a/runtime/datetime/timelib_wrapper.h b/runtime/datetime/timelib_wrapper.h index 51cced348c..5c06638bf5 100644 --- a/runtime/datetime/timelib_wrapper.h +++ b/runtime/datetime/timelib_wrapper.h @@ -2,7 +2,7 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" // php_timelib wraps the https://github.com/derickr/timelib library // which is used in PHP to implement several datetime lib functions. diff --git a/runtime/env.h b/runtime/env.h index 89848bcc83..e2ded6ae20 100644 --- a/runtime/env.h +++ b/runtime/env.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" array f$getenv() noexcept; Optional f$getenv(const string &varname, bool local_only = false) noexcept; diff --git a/runtime/exception.cpp b/runtime/exception.cpp index 3cc90786e8..e6a3dfa238 100644 --- a/runtime/exception.cpp +++ b/runtime/exception.cpp @@ -6,6 +6,7 @@ #include "common/fast-backtrace.h" +#include "runtime/context/runtime-context.h" #include "runtime/critical_section.h" #include "runtime/string_functions.h" @@ -86,16 +87,16 @@ Exception new_Exception(const string &file, int64_t line, const string &message, } Exception f$err(const string &file, int64_t line, const string &code, const string &desc) { - return new_Exception(file, line, (static_SB.clean() << "ERR_" << code << ": " << desc).str(), 0); + return new_Exception(file, line, (kphp_runtime_context.static_SB.clean() << "ERR_" << code << ": " << desc).str(), 0); } string exception_trace_as_string(const Throwable &e) { - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); for (int64_t i = 0; i < e->trace.count(); i++) { array current = e->trace.get_value(i); - static_SB << '#' << i << ' ' << current.get_value(string("file", 4)) << ": " << current.get_value(string("function", 8)) << "\n"; + kphp_runtime_context.static_SB << '#' << i << ' ' << current.get_value(string("file", 4)) << ": " << current.get_value(string("function", 8)) << "\n"; } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } void exception_initialize(const Throwable &e, const string &message, int64_t code) { diff --git a/runtime/exception.h b/runtime/exception.h index 9fd7047f80..85b8ebaedd 100644 --- a/runtime/exception.h +++ b/runtime/exception.h @@ -8,12 +8,12 @@ #include "common/algorithms/hashes.h" #include "common/wrappers/string_view.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" #include "runtime/instance-copy-processor.h" -#include "runtime/to-array-processor.h" -#include "runtime/kphp_core.h" #include "runtime/memory_usage.h" -#include "runtime/refcountable_php_classes.h" +#include "runtime/to-array-processor.h" array> f$debug_backtrace(); diff --git a/runtime/exec.h b/runtime/exec.h index 089f1747c8..5539a777f0 100644 --- a/runtime/exec.h +++ b/runtime/exec.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" int64_t &get_dummy_result_code() noexcept; diff --git a/runtime/ffi.h b/runtime/ffi.h index 781865957c..53ac0fc837 100644 --- a/runtime/ffi.h +++ b/runtime/ffi.h @@ -4,10 +4,10 @@ #pragma once +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/kphp_core.h" #include "runtime/memory_usage.h" -#include "runtime/refcountable_php_classes.h" template struct C$FFI$CData: public refcountable_php_classes>, private DummyVisitorMethods { diff --git a/runtime/files.cpp b/runtime/files.cpp index 01f38ffa83..b33af43652 100644 --- a/runtime/files.cpp +++ b/runtime/files.cpp @@ -17,11 +17,12 @@ #include "common/macos-ports.h" #include "common/wrappers/mkdir_recursive.h" +#include "runtime/context/runtime-context.h" #include "runtime/critical_section.h" #include "runtime/interface.h" #include "runtime/kphp_tracing.h" #include "runtime/streams.h" -#include "runtime/string_functions.h"//php_buf, TODO +#include "runtime/string_functions.h" //php_buf, TODO static int32_t opened_fd{-1}; @@ -477,7 +478,7 @@ static Optional full_realpath(const string &path) { // realpath resolvin const char *basename_c_str = __xpg_basename(basename_path_copy.buffer()); dl::leave_critical_section(); - return result_cache = (static_SB.clean() << file_wrapper_name << real_path << '/' << basename_c_str).str(); + return result_cache = (kphp_runtime_context.static_SB.clean() << file_wrapper_name << real_path << '/' << basename_c_str).str(); } result_cache = LETTER_a; return false; diff --git a/runtime/files.h b/runtime/files.h index 5a4373bf5b..58654430ed 100644 --- a/runtime/files.h +++ b/runtime/files.h @@ -8,7 +8,7 @@ #include #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" extern const string LETTER_a; diff --git a/runtime/from-json-processor.h b/runtime/from-json-processor.h index 7e77cc8c8f..3c54975c3a 100644 --- a/runtime/from-json-processor.h +++ b/runtime/from-json-processor.h @@ -6,7 +6,7 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/json-functions.h" #include "runtime/json-processor-utils.h" @@ -91,13 +91,13 @@ class FromJsonVisitor { } void do_set(JsonRawString &value, const mixed &json) noexcept { - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); if (!impl_::JsonEncoder{0, false, get_json_obj_magic_key()}.encode(json)) { error_.append("failed to decode @kphp-json raw_string field "); error_.append(json_path_.to_string()); return; } - value.str = static_SB.str(); + value.str = kphp_runtime_context.static_SB.str(); } template diff --git a/runtime/instance-cache.cpp b/runtime/instance-cache.cpp index 04d034b67f..2bae5d31d9 100644 --- a/runtime/instance-cache.cpp +++ b/runtime/instance-cache.cpp @@ -13,12 +13,12 @@ #include "common/kprintf.h" #include "common/wrappers/memory-utils.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/memory-resource/resource_allocator.h" #include "runtime/allocator.h" #include "runtime/critical_section.h" #include "runtime/inter-process-mutex.h" #include "runtime/inter-process-resource.h" -#include "runtime/memory_resource/resource_allocator.h" -#include "runtime/refcountable_php_classes.h" namespace impl_ { diff --git a/runtime/instance-cache.h b/runtime/instance-cache.h index 1590ffaccf..b9b1706673 100644 --- a/runtime/instance-cache.h +++ b/runtime/instance-cache.h @@ -18,10 +18,9 @@ #include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" #include "runtime/instance-copy-processor.h" -#include "runtime/kphp_core.h" #include "runtime/memory_usage.h" -#include "runtime/shape.h" #include "server/statshouse/statshouse-manager.h" enum class InstanceCacheOpStatus; diff --git a/runtime/instance-copy-processor.h b/runtime/instance-copy-processor.h index 696e74687b..eac8457c3e 100644 --- a/runtime/instance-copy-processor.h +++ b/runtime/instance-copy-processor.h @@ -10,10 +10,10 @@ #include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" #include "runtime/allocator.h" #include "runtime/critical_section.h" -#include "runtime/kphp_core.h" -#include "runtime/memory_resource/unsynchronized_pool_resource.h" namespace impl_ { diff --git a/runtime/interface.cpp b/runtime/interface.cpp index be036860e7..76f75e837c 100644 --- a/runtime/interface.cpp +++ b/runtime/interface.cpp @@ -24,6 +24,7 @@ #include "runtime/array_functions.h" #include "runtime/bcmath.h" #include "runtime/confdata-functions.h" +#include "runtime/context/runtime-context.h" #include "runtime/critical_section.h" #include "runtime/curl.h" #include "runtime/datetime/datetime_functions.h" @@ -374,29 +375,29 @@ void f$send_http_103_early_hints(const array & headers) { void f$setrawcookie(const string &name, const string &value, int64_t expire, const string &path, const string &domain, bool secure, bool http_only) { string date = f$gmdate(HTTP_DATE, expire); - static_SB_spare.clean() << "Set-Cookie: " << name << '='; + kphp_runtime_context.static_SB_spare.clean() << "Set-Cookie: " << name << '='; if (value.empty()) { - static_SB_spare << "DELETED; expires=Thu, 01 Jan 1970 00:00:01 GMT"; + kphp_runtime_context.static_SB_spare << "DELETED; expires=Thu, 01 Jan 1970 00:00:01 GMT"; } else { - static_SB_spare << value; + kphp_runtime_context.static_SB_spare << value; if (expire != 0) { - static_SB_spare << "; expires=" << date; + kphp_runtime_context.static_SB_spare << "; expires=" << date; } } if (!path.empty()) { - static_SB_spare << "; path=" << path; + kphp_runtime_context.static_SB_spare << "; path=" << path; } if (!domain.empty()) { - static_SB_spare << "; domain=" << domain; + kphp_runtime_context.static_SB_spare << "; domain=" << domain; } if (secure) { - static_SB_spare << "; secure"; + kphp_runtime_context.static_SB_spare << "; secure"; } if (http_only) { - static_SB_spare << "; HttpOnly"; + kphp_runtime_context.static_SB_spare << "; HttpOnly"; } - header(static_SB_spare.c_str(), (int)static_SB_spare.size(), false); + header(kphp_runtime_context.static_SB_spare.c_str(), (int)kphp_runtime_context.static_SB_spare.size(), false); } void f$setcookie(const string &name, const string &value, int64_t expire, const string &path, const string &domain, bool secure, bool http_only) { @@ -486,32 +487,32 @@ static inline const char *http_get_error_msg_text(int *code) { } static void set_content_length_header(int content_length) { - static_SB_spare.clean() << "Content-Length: " << content_length; - header(static_SB_spare.c_str(), (int)static_SB_spare.size()); + kphp_runtime_context.static_SB_spare.clean() << "Content-Length: " << content_length; + header(kphp_runtime_context.static_SB_spare.c_str(), (int)kphp_runtime_context.static_SB_spare.size()); } -static const string_buffer *get_headers() {//can't use static_SB, returns pointer to static_SB_spare +static const string_buffer *get_headers() {//can't use static_SB, returns pointer to kphp_runtime_context.static_SB_spare string date = f$gmdate(HTTP_DATE); - static_SB_spare.clean() << "Date: " << date; - header(static_SB_spare.c_str(), (int)static_SB_spare.size()); + kphp_runtime_context.static_SB_spare.clean() << "Date: " << date; + header(kphp_runtime_context.static_SB_spare.c_str(), (int)kphp_runtime_context.static_SB_spare.size()); php_assert (dl::query_num == header_last_query_num); - static_SB_spare.clean(); + kphp_runtime_context.static_SB_spare.clean(); if (!http_status_line.empty()) { - static_SB_spare << http_status_line << "\r\n"; + kphp_runtime_context.static_SB_spare << http_status_line << "\r\n"; } else { const char *message = http_get_error_msg_text(&http_return_code); - static_SB_spare << "HTTP/1.1 " << http_return_code << " " << message << "\r\n"; + kphp_runtime_context.static_SB_spare << "HTTP/1.1 " << http_return_code << " " << message << "\r\n"; } const array *arr = headers; for (array::const_iterator p = arr->begin(); p != arr->end(); ++p) { - static_SB_spare << p.get_value(); + kphp_runtime_context.static_SB_spare << p.get_value(); } - static_SB_spare << "\r\n"; + kphp_runtime_context.static_SB_spare << "\r\n"; - return &static_SB_spare; + return &kphp_runtime_context.static_SB_spare; } constexpr uint32_t MAX_SHUTDOWN_FUNCTIONS = 256; @@ -571,7 +572,7 @@ void f$flush() { http_send_immediate_response(http_headers ? http_headers->buffer() : nullptr, http_headers ? http_headers->size() : 0, http_body->buffer(), http_body->size()); oub[ob_system_level].clean(); - static_SB_spare.clean(); + kphp_runtime_context.static_SB_spare.clean(); } void f$fastcgi_finish_request(int64_t exit_code) { @@ -766,14 +767,14 @@ double f$thread_pool_test_load(int64_t size, int64_t n, double a, double b) { } string f$long2ip(int64_t num) { - static_SB.clean().reserve(100); + kphp_runtime_context.static_SB.clean().reserve(100); for (int i = 3; i >= 0; i--) { - static_SB << ((num >> (i * 8)) & 255); + kphp_runtime_context.static_SB << ((num >> (i * 8)) & 255); if (i) { - static_SB.append_char('.'); + kphp_runtime_context.static_SB.append_char('.'); } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } Optional> f$gethostbynamel(const string &name) { @@ -1561,7 +1562,7 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q if (http_data.uri) { if (http_data.get_len) { - superglobals.v$_SERVER.set_value(string("REQUEST_URI"), (static_SB.clean() << uri_str << '?' << get_str).str()); + superglobals.v$_SERVER.set_value(string("REQUEST_URI"), (kphp_runtime_context.static_SB.clean() << uri_str << '?' << get_str).str()); } else { superglobals.v$_SERVER.set_value(string("REQUEST_URI"), uri_str); } @@ -1740,7 +1741,7 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q superglobals.v$_SERVER.set_value(string("REQUEST_TIME_FLOAT"), cur_time); superglobals.v$_SERVER.set_value(string("SERVER_PORT"), string("80")); superglobals.v$_SERVER.set_value(string("SERVER_PROTOCOL"), string("HTTP/1.1")); - superglobals.v$_SERVER.set_value(string("SERVER_SIGNATURE"), (static_SB.clean() << "Apache/2.2.9 (Debian) PHP/5.2.6-1<(static_buffer_length_limit)); - init_interface_lib(); } @@ -2386,7 +2385,6 @@ static void free_runtime_libs() { free_instance_cache_lib(); free_kphp_backtrace(); - free_migration_php8(); free_use_updated_gmmktime(); free_detect_incorrect_encoding_names(); @@ -2425,7 +2423,7 @@ void global_init_script_allocator() { } void init_runtime_environment(const php_query_data_t &data, PhpScriptBuiltInSuperGlobals &superglobals, void *mem, size_t script_mem_size, size_t oom_handling_mem_size) { - dl::init_script_allocator(mem, script_mem_size, oom_handling_mem_size); + kphp_runtime_context.init(mem, script_mem_size, oom_handling_mem_size); reset_global_interface_vars(superglobals); init_runtime_libs(); init_superglobals(data, superglobals); @@ -2435,7 +2433,7 @@ void free_runtime_environment(PhpScriptBuiltInSuperGlobals &superglobals) { reset_superglobals(superglobals); free_runtime_libs(); reset_global_interface_vars(superglobals); - dl::free_script_allocator(); + kphp_runtime_context.free(); } void worker_global_init(WorkerType worker_type) noexcept { diff --git a/runtime/interface.h b/runtime/interface.h index 7b21347b32..b1a05f8e55 100644 --- a/runtime/interface.h +++ b/runtime/interface.h @@ -8,9 +8,8 @@ #include "common/wrappers/string_view.h" +#include "runtime-core/runtime-core.h" #include "runtime/critical_section.h" -#include "runtime/kphp_core.h" -#include "runtime/optional.h" #include "runtime/php-script-globals.h" #include "server/php-query-data.h" #include "server/statshouse/statshouse-manager.h" diff --git a/runtime/job-workers/client-functions.h b/runtime/job-workers/client-functions.h index 95f03895d0..9c4730da30 100644 --- a/runtime/job-workers/client-functions.h +++ b/runtime/job-workers/client-functions.h @@ -4,8 +4,8 @@ #pragma once +#include "runtime-core/runtime-core.h" #include "runtime/job-workers/job-interface.h" -#include "runtime/kphp_core.h" void free_job_client_interface_lib() noexcept; diff --git a/runtime/job-workers/job-interface.h b/runtime/job-workers/job-interface.h index 8a429591dc..16ef3376b9 100644 --- a/runtime/job-workers/job-interface.h +++ b/runtime/job-workers/job-interface.h @@ -7,10 +7,10 @@ #include "common/algorithms/hashes.h" #include "common/wrappers/string_view.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/instance-copy-processor.h" #include "runtime/to-array-processor.h" -#include "runtime/kphp_core.h" -#include "runtime/refcountable_php_classes.h" class InstanceReferencesCountingVisitor; diff --git a/runtime/job-workers/processing-jobs.h b/runtime/job-workers/processing-jobs.h index a68eb30d4e..bea2ad435b 100644 --- a/runtime/job-workers/processing-jobs.h +++ b/runtime/job-workers/processing-jobs.h @@ -7,7 +7,7 @@ #include "common/mixin/not_copyable.h" #include "common/smart_ptrs/singleton.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/net_events.h" #include "runtime/job-workers/job-interface.h" diff --git a/runtime/job-workers/server-functions.h b/runtime/job-workers/server-functions.h index c4380ebb65..868ec5c377 100644 --- a/runtime/job-workers/server-functions.h +++ b/runtime/job-workers/server-functions.h @@ -4,8 +4,8 @@ #pragma once +#include "runtime-core/runtime-core.h" #include "runtime/job-workers/job-interface.h" -#include "runtime/kphp_core.h" struct job_query_data; void init_job_server_interface_lib(job_query_data job_data) noexcept; diff --git a/runtime/json-functions.cpp b/runtime/json-functions.cpp index 79c958ca04..c066304e23 100644 --- a/runtime/json-functions.cpp +++ b/runtime/json-functions.cpp @@ -14,12 +14,12 @@ namespace { void json_append_one_char(unsigned int c) noexcept { - static_SB.append_char('\\'); - static_SB.append_char('u'); - static_SB.append_char("0123456789abcdef"[c >> 12]); - static_SB.append_char("0123456789abcdef"[(c >> 8) & 15]); - static_SB.append_char("0123456789abcdef"[(c >> 4) & 15]); - static_SB.append_char("0123456789abcdef"[c & 15]); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('u'); + kphp_runtime_context.static_SB.append_char("0123456789abcdef"[c >> 12]); + kphp_runtime_context.static_SB.append_char("0123456789abcdef"[(c >> 8) & 15]); + kphp_runtime_context.static_SB.append_char("0123456789abcdef"[(c >> 4) & 15]); + kphp_runtime_context.static_SB.append_char("0123456789abcdef"[c & 15]); } bool json_append_char(unsigned int c) noexcept { @@ -41,54 +41,54 @@ bool json_append_char(unsigned int c) noexcept { bool do_json_encode_string_php(const JsonPath &json_path, const char *s, int len, int64_t options) noexcept { - int begin_pos = static_SB.size(); + int begin_pos = kphp_runtime_context.static_SB.size(); if (options & JSON_UNESCAPED_UNICODE) { - static_SB.reserve(2 * len + 2); + kphp_runtime_context.static_SB.reserve(2 * len + 2); } else { - static_SB.reserve(6 * len + 2); + kphp_runtime_context.static_SB.reserve(6 * len + 2); } - static_SB.append_char('"'); + kphp_runtime_context.static_SB.append_char('"'); auto fire_error = [json_path, begin_pos](int pos) { php_warning("%s: Not a valid utf-8 character at pos %d in function json_encode", json_path.to_string().c_str(), pos); - static_SB.set_pos(begin_pos); - static_SB.append("null", 4); + kphp_runtime_context.static_SB.set_pos(begin_pos); + kphp_runtime_context.static_SB.append("null", 4); return false; }; for (int pos = 0; pos < len; pos++) { switch (s[pos]) { case '"': - static_SB.append_char('\\'); - static_SB.append_char('"'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('"'); break; case '\\': - static_SB.append_char('\\'); - static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('\\'); break; case '/': - static_SB.append_char('\\'); - static_SB.append_char('/'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('/'); break; case '\b': - static_SB.append_char('\\'); - static_SB.append_char('b'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('b'); break; case '\f': - static_SB.append_char('\\'); - static_SB.append_char('f'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('f'); break; case '\n': - static_SB.append_char('\\'); - static_SB.append_char('n'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('n'); break; case '\r': - static_SB.append_char('\\'); - static_SB.append_char('r'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('r'); break; case '\t': - static_SB.append_char('\\'); - static_SB.append_char('t'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('t'); break; case 0 ... 7: case 11: @@ -110,8 +110,8 @@ bool do_json_encode_string_php(const JsonPath &json_path, const char *s, int len return fire_error(pos); } if (options & JSON_UNESCAPED_UNICODE) { - static_SB.append_char(static_cast(a)); - static_SB.append_char(static_cast(b)); + kphp_runtime_context.static_SB.append_char(static_cast(a)); + kphp_runtime_context.static_SB.append_char(static_cast(b)); } else if (!json_append_char(((a & 0x1f) << 6) | (b & 0x3f))) { return fire_error(pos); } @@ -127,9 +127,9 @@ bool do_json_encode_string_php(const JsonPath &json_path, const char *s, int len return fire_error(pos); } if (options & JSON_UNESCAPED_UNICODE) { - static_SB.append_char(static_cast(a)); - static_SB.append_char(static_cast(b)); - static_SB.append_char(static_cast(c)); + kphp_runtime_context.static_SB.append_char(static_cast(a)); + kphp_runtime_context.static_SB.append_char(static_cast(b)); + kphp_runtime_context.static_SB.append_char(static_cast(c)); } else if (!json_append_char(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f))) { return fire_error(pos); } @@ -145,10 +145,10 @@ bool do_json_encode_string_php(const JsonPath &json_path, const char *s, int len return fire_error(pos); } if (options & JSON_UNESCAPED_UNICODE) { - static_SB.append_char(static_cast(a)); - static_SB.append_char(static_cast(b)); - static_SB.append_char(static_cast(c)); - static_SB.append_char(static_cast(d)); + kphp_runtime_context.static_SB.append_char(static_cast(a)); + kphp_runtime_context.static_SB.append_char(static_cast(b)); + kphp_runtime_context.static_SB.append_char(static_cast(c)); + kphp_runtime_context.static_SB.append_char(static_cast(d)); } else if (!json_append_char(((a & 0x07) << 18) | ((b & 0x3f) << 12) | ((c & 0x3f) << 6) | (d & 0x3f))) { return fire_error(pos); } @@ -158,57 +158,57 @@ bool do_json_encode_string_php(const JsonPath &json_path, const char *s, int len return fire_error(pos); } default: - static_SB.append_char(s[pos]); + kphp_runtime_context.static_SB.append_char(s[pos]); break; } } - static_SB.append_char('"'); + kphp_runtime_context.static_SB.append_char('"'); return true; } bool do_json_encode_string_vkext(const char *s, int len) noexcept { - static_SB.reserve(2 * len + 2); - if (static_SB.string_buffer_error_flag == STRING_BUFFER_ERROR_FLAG_FAILED) { + kphp_runtime_context.static_SB.reserve(2 * len + 2); + if (kphp_runtime_context.sb_lib_context.error_flag == STRING_BUFFER_ERROR_FLAG_FAILED) { return false; } - static_SB.append_char('"'); + kphp_runtime_context.static_SB.append_char('"'); for (int pos = 0; pos < len; pos++) { char c = s[pos]; if (unlikely (static_cast(c) < 32u)) { switch (c) { case '\b': - static_SB.append_char('\\'); - static_SB.append_char('b'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('b'); break; case '\f': - static_SB.append_char('\\'); - static_SB.append_char('f'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('f'); break; case '\n': - static_SB.append_char('\\'); - static_SB.append_char('n'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('n'); break; case '\r': - static_SB.append_char('\\'); - static_SB.append_char('r'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('r'); break; case '\t': - static_SB.append_char('\\'); - static_SB.append_char('t'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('t'); break; } } else { if (c == '"' || c == '\\' || c == '/') { - static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('\\'); } - static_SB.append_char(c); + kphp_runtime_context.static_SB.append_char(c); } } - static_SB.append_char('"'); + kphp_runtime_context.static_SB.append_char('"'); return true; } @@ -254,20 +254,20 @@ JsonEncoder::JsonEncoder(int64_t options, bool simple_encode, const char *json_o bool JsonEncoder::encode(bool b) noexcept { if (b) { - static_SB.append("true", 4); + kphp_runtime_context.static_SB.append("true", 4); } else { - static_SB.append("false", 5); + kphp_runtime_context.static_SB.append("false", 5); } return true; } bool JsonEncoder::encode_null() const noexcept { - static_SB.append("null", 4); + kphp_runtime_context.static_SB.append("null", 4); return true; } bool JsonEncoder::encode(int64_t i) noexcept { - static_SB << i; + kphp_runtime_context.static_SB << i; return true; } @@ -275,12 +275,12 @@ bool JsonEncoder::encode(double d) noexcept { if (vk::any_of_equal(std::fpclassify(d), FP_INFINITE, FP_NAN)) { php_warning("%s: strange double %lf in function json_encode", json_path_.to_string().c_str(), d); if (options_ & JSON_PARTIAL_OUTPUT_ON_ERROR) { - static_SB.append("0", 1); + kphp_runtime_context.static_SB.append("0", 1); } else { return false; } } else { - static_SB << (simple_encode_ ? f$number_format(d, 6, string{"."}, string{}) : string{d}); + kphp_runtime_context.static_SB << (simple_encode_ ? f$number_format(d, 6, string{"."}, string{}) : string{d}); } return true; } diff --git a/runtime/json-functions.h b/runtime/json-functions.h index 8754d73dd5..8e46c9418d 100644 --- a/runtime/json-functions.h +++ b/runtime/json-functions.h @@ -4,8 +4,9 @@ #pragma once +#include "runtime-core/runtime-core.h" +#include "runtime/context/runtime-context.h" #include "runtime/exception.h" -#include "runtime/kphp_core.h" #include @@ -80,14 +81,14 @@ bool JsonEncoder::encode(const array &arr) noexcept { } is_vector &= !force_object; - static_SB << "{["[is_vector]; + kphp_runtime_context.static_SB << "{["[is_vector]; if (is_vector) { int i = 0; json_path_.enter(nullptr); // similar key for all entries for (auto p : arr) { if (i != 0) { - static_SB << ','; + kphp_runtime_context.static_SB << ','; } if (!encode(p.get_value())) { if (!(options_ & JSON_PARTIAL_OUTPUT_ON_ERROR)) { @@ -101,7 +102,7 @@ bool JsonEncoder::encode(const array &arr) noexcept { bool is_first = true; for (auto p : arr) { if (!is_first) { - static_SB << ','; + kphp_runtime_context.static_SB << ','; } is_first = false; const char *next_key = nullptr; @@ -109,7 +110,7 @@ bool JsonEncoder::encode(const array &arr) noexcept { if (array::is_int_key(key)) { auto int_key = key.to_int(); next_key = nullptr; - static_SB << '"' << int_key << '"'; + kphp_runtime_context.static_SB << '"' << int_key << '"'; } else { const string &str_key = key.as_string(); // skip service key intended only for distinguish empty json object with empty json array @@ -123,7 +124,7 @@ bool JsonEncoder::encode(const array &arr) noexcept { } } } - static_SB << ':'; + kphp_runtime_context.static_SB << ':'; json_path_.enter(next_key); if (!encode(p.get_value())) { if (!(options_ & JSON_PARTIAL_OUTPUT_ON_ERROR)) { @@ -134,7 +135,7 @@ bool JsonEncoder::encode(const array &arr) noexcept { } } - static_SB << "}]"[is_vector]; + kphp_runtime_context.static_SB << "}]"[is_vector]; return true; } @@ -161,26 +162,26 @@ Optional f$json_encode(const T &v, int64_t options = 0, bool simple_enco return false; } - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); if (unlikely(!impl_::JsonEncoder(options, simple_encode).encode(v))) { return false; } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } template string f$vk_json_encode_safe(const T &v, bool simple_encode = true) noexcept { - static_SB.clean(); - string_buffer::string_buffer_error_flag = STRING_BUFFER_ERROR_FLAG_ON; + kphp_runtime_context.static_SB.clean(); + kphp_runtime_context.sb_lib_context.error_flag = STRING_BUFFER_ERROR_FLAG_ON; impl_::JsonEncoder(0, simple_encode).encode(v); - if (unlikely(string_buffer::string_buffer_error_flag == STRING_BUFFER_ERROR_FLAG_FAILED)) { - static_SB.clean(); - string_buffer::string_buffer_error_flag = STRING_BUFFER_ERROR_FLAG_OFF; + if (unlikely(kphp_runtime_context.sb_lib_context.error_flag == STRING_BUFFER_ERROR_FLAG_FAILED)) { + kphp_runtime_context.static_SB.clean(); + kphp_runtime_context.sb_lib_context.error_flag = STRING_BUFFER_ERROR_FLAG_OFF; THROW_EXCEPTION (new_Exception(string(__FILE__), __LINE__, string("json_encode buffer overflow", 27))); return {}; } - string_buffer::string_buffer_error_flag = STRING_BUFFER_ERROR_FLAG_OFF; - return static_SB.str(); + kphp_runtime_context.sb_lib_context.error_flag = STRING_BUFFER_ERROR_FLAG_OFF; + return kphp_runtime_context.static_SB.str(); } template diff --git a/runtime/json-writer.cpp b/runtime/json-writer.cpp index 065477f377..2e9ef7cdbc 100644 --- a/runtime/json-writer.cpp +++ b/runtime/json-writer.cpp @@ -5,6 +5,7 @@ #include "runtime/json-writer.h" #include "runtime/array_functions.h" +#include "runtime/context/runtime-context.h" #include "runtime/math_functions.h" // note: json-writer.cpp is used for classes, e.g. `JsonEncoder::encode(new A)` (also see from/to visitors) @@ -56,18 +57,18 @@ static void escape_json_string(string_buffer &buffer, std::string_view s) noexce JsonWriter::JsonWriter(bool pretty_print, bool preserve_zero_fraction) noexcept : pretty_print_(pretty_print) , preserve_zero_fraction_(preserve_zero_fraction) { - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); } JsonWriter::~JsonWriter() noexcept { - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); } bool JsonWriter::write_bool(bool b) noexcept { if (!register_value()) { return false; } - b ? static_SB.append("true", 4) : static_SB.append("false", 5); + b ? kphp_runtime_context.static_SB.append("true", 4) : kphp_runtime_context.static_SB.append("false", 5); return true; } @@ -75,7 +76,7 @@ bool JsonWriter::write_int(int64_t i) noexcept { if (!register_value()) { return false; } - static_SB << i; + kphp_runtime_context.static_SB << i; return true; } @@ -87,13 +88,13 @@ bool JsonWriter::write_double(double d) noexcept { d = 0.0; } if (double_precision_) { - static_SB << f$round(d, double_precision_); + kphp_runtime_context.static_SB << f$round(d, double_precision_); } else { - static_SB << d; + kphp_runtime_context.static_SB << d; } if (preserve_zero_fraction_) { if (double dummy = 0.0; std::modf(d, &dummy) == 0.0) { - static_SB << ".0"; + kphp_runtime_context.static_SB << ".0"; } } return true; @@ -103,10 +104,10 @@ bool JsonWriter::write_string(const string &s) noexcept { if (!register_value()) { return false; } - static_SB.reserve(2 * s.size() + 2); - static_SB.append_char('"'); - escape_json_string(static_SB, {s.c_str(), s.size()}); - static_SB.append_char('"'); + kphp_runtime_context.static_SB.reserve(2 * s.size() + 2); + kphp_runtime_context.static_SB.append_char('"'); + escape_json_string(kphp_runtime_context.static_SB, {s.c_str(), s.size()}); + kphp_runtime_context.static_SB.append_char('"'); return true; } @@ -114,7 +115,7 @@ bool JsonWriter::write_raw_string(const string &s) noexcept { if (!register_value()) { return false; } - static_SB << s; + kphp_runtime_context.static_SB << s; return true; } @@ -122,7 +123,7 @@ bool JsonWriter::write_null() noexcept { if (!register_value()) { return false; } - static_SB.append("null", 4); + kphp_runtime_context.static_SB.append("null", 4); return true; } @@ -132,22 +133,22 @@ bool JsonWriter::write_key(std::string_view key, bool escape) noexcept { return false; } if (stack_.back().values_count) { - static_SB << ','; + kphp_runtime_context.static_SB << ','; } if (pretty_print_) { - static_SB << '\n'; + kphp_runtime_context.static_SB << '\n'; write_indent(); } - static_SB << '"'; + kphp_runtime_context.static_SB << '"'; if (escape) { - escape_json_string(static_SB, key); + escape_json_string(kphp_runtime_context.static_SB, key); } else { - static_SB.append(key.data(), key.size()); + kphp_runtime_context.static_SB.append(key.data(), key.size()); } - static_SB << '"'; - static_SB << ':'; + kphp_runtime_context.static_SB << '"'; + kphp_runtime_context.static_SB << ':'; if (pretty_print_) { - static_SB << ' '; + kphp_runtime_context.static_SB << ' '; } return true; } @@ -177,7 +178,7 @@ string JsonWriter::get_error() const noexcept { } string JsonWriter::get_final_json() const noexcept { - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } bool JsonWriter::new_level(bool is_array) noexcept { @@ -186,7 +187,7 @@ bool JsonWriter::new_level(bool is_array) noexcept { } stack_.emplace_back(NestedLevel{.in_array = is_array}); - static_SB << (is_array ? '[' : '{'); + kphp_runtime_context.static_SB << (is_array ? '[' : '{'); indent_ += 4; return true; } @@ -208,11 +209,11 @@ bool JsonWriter::exit_level(bool is_array) noexcept { indent_ -= 4; if (pretty_print_ && cur_level.values_count) { - static_SB << '\n'; + kphp_runtime_context.static_SB << '\n'; write_indent(); } - static_SB << (is_array ? ']' : '}'); + kphp_runtime_context.static_SB << (is_array ? ']' : '}'); return true; } @@ -228,10 +229,10 @@ bool JsonWriter::register_value() noexcept { auto &top = stack_.back(); if (top.in_array) { if (top.values_count) { - static_SB << ','; + kphp_runtime_context.static_SB << ','; } if (pretty_print_) { - static_SB << '\n'; + kphp_runtime_context.static_SB << '\n'; write_indent(); } } @@ -242,9 +243,9 @@ bool JsonWriter::register_value() noexcept { void JsonWriter::write_indent() const noexcept { if (indent_) { - static_SB.reserve(indent_); + kphp_runtime_context.static_SB.reserve(indent_); for (std::size_t i = 0; i < indent_; ++i) { - static_SB.append_char(' '); + kphp_runtime_context.static_SB.append_char(' '); } } } diff --git a/runtime/json-writer.h b/runtime/json-writer.h index 76bfd8601b..507716dbf9 100644 --- a/runtime/json-writer.h +++ b/runtime/json-writer.h @@ -4,7 +4,8 @@ #pragma once -#include "runtime/kphp_core.h" +#include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" #include diff --git a/runtime/kphp-backtrace.h b/runtime/kphp-backtrace.h index 9207ca5e32..9c02b3eacc 100644 --- a/runtime/kphp-backtrace.h +++ b/runtime/kphp-backtrace.h @@ -8,8 +8,8 @@ #include #include "common/wrappers/iterator_range.h" - -#include "runtime/kphp_core.h" +#include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" class KphpBacktrace : vk::not_copyable { public: diff --git a/runtime/kphp_ml/kphp_ml.cpp b/runtime/kphp_ml/kphp_ml.cpp index b73c775969..622083d72a 100644 --- a/runtime/kphp_ml/kphp_ml.cpp +++ b/runtime/kphp_ml/kphp_ml.cpp @@ -6,8 +6,8 @@ // This file exists both in KPHP and in a private vkcom repo "ml_experiments". // They are almost identical, besides include paths and input types (`array` vs `unordered_map`). -#include "runtime/kphp_core.h" #include "runtime/kphp_ml/kphp_ml.h" +#include "runtime-core/runtime-core.h" // for detailed comments about KML, see kphp_ml.h diff --git a/runtime/kphp_ml/kphp_ml_catboost.cpp b/runtime/kphp_ml/kphp_ml_catboost.cpp index 24ccaa712e..dae506e7de 100644 --- a/runtime/kphp_ml/kphp_ml_catboost.cpp +++ b/runtime/kphp_ml/kphp_ml_catboost.cpp @@ -8,7 +8,7 @@ #include "runtime/kphp_ml/kphp_ml_catboost.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/kphp_ml/kphp_ml.h" /* diff --git a/runtime/kphp_ml/kphp_ml_catboost.h b/runtime/kphp_ml/kphp_ml_catboost.h index 2214225587..94ad2ee35b 100644 --- a/runtime/kphp_ml/kphp_ml_catboost.h +++ b/runtime/kphp_ml/kphp_ml_catboost.h @@ -12,7 +12,7 @@ #include #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" namespace kphp_ml { struct MLModel; } diff --git a/runtime/kphp_ml/kphp_ml_init.h b/runtime/kphp_ml/kphp_ml_init.h index 4843f785c8..c1648299da 100644 --- a/runtime/kphp_ml/kphp_ml_init.h +++ b/runtime/kphp_ml/kphp_ml_init.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" namespace kphp_ml { struct MLModel; } diff --git a/runtime/kphp_ml/kphp_ml_interface.h b/runtime/kphp_ml/kphp_ml_interface.h index bb785d2173..483732a3e3 100644 --- a/runtime/kphp_ml/kphp_ml_interface.h +++ b/runtime/kphp_ml/kphp_ml_interface.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" Optional> f$kml_xgboost_predict_matrix(const string &model_name, const array> &features_map_matrix); diff --git a/runtime/kphp_ml/kphp_ml_xgboost.cpp b/runtime/kphp_ml/kphp_ml_xgboost.cpp index 82764ba0ad..aa87d6f0a4 100644 --- a/runtime/kphp_ml/kphp_ml_xgboost.cpp +++ b/runtime/kphp_ml/kphp_ml_xgboost.cpp @@ -10,7 +10,7 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/kphp_ml/kphp_ml.h" /* diff --git a/runtime/kphp_ml/kphp_ml_xgboost.h b/runtime/kphp_ml/kphp_ml_xgboost.h index bd124ce77d..bdad91a1ba 100644 --- a/runtime/kphp_ml/kphp_ml_xgboost.h +++ b/runtime/kphp_ml/kphp_ml_xgboost.h @@ -11,7 +11,7 @@ #include #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" /* * For detailed comments about KML, see kphp_ml.h. diff --git a/runtime/kphp_tracing.h b/runtime/kphp_tracing.h index 3517d6288c..420fcb33fd 100644 --- a/runtime/kphp_tracing.h +++ b/runtime/kphp_tracing.h @@ -6,9 +6,9 @@ #include +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/critical_section.h" -#include "runtime/kphp_core.h" -#include "runtime/refcountable_php_classes.h" #include "runtime/dummy-visitor-methods.h" // for detailed comments about tracing in general, see kphp_tracing.cpp diff --git a/runtime/kphp_tracing_binlog.cpp b/runtime/kphp_tracing_binlog.cpp index ed09c0b7bc..03c65672ea 100644 --- a/runtime/kphp_tracing_binlog.cpp +++ b/runtime/kphp_tracing_binlog.cpp @@ -6,8 +6,11 @@ #include #include +#include +#include #include "runtime/critical_section.h" +#include "runtime/allocator.h" #include "server/json-logger.h" diff --git a/runtime/kphp_tracing_binlog.h b/runtime/kphp_tracing_binlog.h index 05f1a65177..b1e334ff80 100644 --- a/runtime/kphp_tracing_binlog.h +++ b/runtime/kphp_tracing_binlog.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" namespace kphp_tracing { diff --git a/runtime/mail.h b/runtime/mail.h index 0ec58241e9..eb1cabf778 100644 --- a/runtime/mail.h +++ b/runtime/mail.h @@ -4,6 +4,6 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" bool f$mail(const string &to, const string &subject, const string &message, string additional_headers = string()); diff --git a/runtime/math_functions.h b/runtime/math_functions.h index 9d71585763..13c02959a4 100644 --- a/runtime/math_functions.h +++ b/runtime/math_functions.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" int64_t f$bindec(const string &number) noexcept; diff --git a/runtime/mbstring.h b/runtime/mbstring.h index 9685f4be76..a6abe3638a 100644 --- a/runtime/mbstring.h +++ b/runtime/mbstring.h @@ -6,7 +6,7 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/string_functions.h" bool mb_UTF8_check(const char *s); diff --git a/runtime/memcache.h b/runtime/memcache.h index fc0c030dfb..048d826241 100644 --- a/runtime/memcache.h +++ b/runtime/memcache.h @@ -6,15 +6,15 @@ #include +#include "common/algorithms/hashes.h" +#include "common/wrappers/string_view.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" #include "runtime/exception.h" -#include "runtime/kphp_core.h" #include "runtime/memory_usage.h" #include "runtime/net_events.h" #include "runtime/resumable.h" #include "runtime/rpc.h" -#include "common/algorithms/hashes.h" -#include "common/wrappers/string_view.h" void init_memcache_lib(); void free_memcache_lib(); diff --git a/runtime/memory_resource/memory_resource.cpp b/runtime/memory_resource/memory_resource.cpp deleted file mode 100644 index a96314e746..0000000000 --- a/runtime/memory_resource/memory_resource.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "runtime/memory_resource/memory_resource.h" - -namespace memory_resource { - -void MemoryStats::write_stats_to(stats_t *stats, const char *prefix) const noexcept { - stats->add_gauge_stat(memory_limit, prefix, ".memory.limit"); - stats->add_gauge_stat(memory_used, prefix, ".memory.used"); - stats->add_gauge_stat(real_memory_used, prefix, ".memory.real_used"); - stats->add_gauge_stat(max_memory_used, prefix, ".memory.used_max"); - stats->add_gauge_stat(max_real_memory_used, prefix, ".memory.real_used_max"); - stats->add_gauge_stat(defragmentation_calls, prefix, ".memory.defragmentation_calls"); - stats->add_gauge_stat(huge_memory_pieces, prefix, ".memory.huge_memory_pieces"); - stats->add_gauge_stat(small_memory_pieces, prefix, ".memory.small_memory_pieces"); -} - -} // namespace memory_resource diff --git a/runtime/memory_resource/dealer.cpp b/runtime/memory_resource_impl/dealer.cpp similarity index 86% rename from runtime/memory_resource/dealer.cpp rename to runtime/memory_resource_impl/dealer.cpp index f9886cbb69..44bcbbb84f 100644 --- a/runtime/memory_resource/dealer.cpp +++ b/runtime/memory_resource_impl/dealer.cpp @@ -2,7 +2,7 @@ // Copyright (c) 2020 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/memory_resource/dealer.h" +#include "runtime/memory_resource_impl//dealer.h" namespace memory_resource { diff --git a/runtime/memory_resource/dealer.h b/runtime/memory_resource_impl/dealer.h similarity index 91% rename from runtime/memory_resource/dealer.h rename to runtime/memory_resource_impl/dealer.h index 958fed3101..e08c5b5ab6 100644 --- a/runtime/memory_resource/dealer.h +++ b/runtime/memory_resource_impl/dealer.h @@ -3,8 +3,8 @@ // Distributed under the GPL v3 License, see LICENSE.notice.txt #pragma once -#include "runtime/memory_resource/heap_resource.h" -#include "runtime/memory_resource/unsynchronized_pool_resource.h" +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" +#include "runtime/memory_resource_impl/heap_resource.h" namespace memory_resource { diff --git a/runtime/memory_resource/heap_resource.cpp b/runtime/memory_resource_impl/heap_resource.cpp similarity index 96% rename from runtime/memory_resource/heap_resource.cpp rename to runtime/memory_resource_impl/heap_resource.cpp index 6fadacd66b..3d395bf1f4 100644 --- a/runtime/memory_resource/heap_resource.cpp +++ b/runtime/memory_resource_impl/heap_resource.cpp @@ -2,7 +2,7 @@ // Copyright (c) 2020 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/memory_resource/heap_resource.h" +#include "runtime/memory_resource_impl//heap_resource.h" #include #include diff --git a/runtime/memory_resource/heap_resource.h b/runtime/memory_resource_impl/heap_resource.h similarity index 90% rename from runtime/memory_resource/heap_resource.h rename to runtime/memory_resource_impl/heap_resource.h index fef4237def..740b043622 100644 --- a/runtime/memory_resource/heap_resource.h +++ b/runtime/memory_resource_impl/heap_resource.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/memory_resource/memory_resource.h" +#include "runtime-core/memory-resource/memory_resource.h" namespace memory_resource { diff --git a/runtime/memory_resource_impl/memory_resource_stats.cpp b/runtime/memory_resource_impl/memory_resource_stats.cpp new file mode 100644 index 0000000000..92db1cce56 --- /dev/null +++ b/runtime/memory_resource_impl/memory_resource_stats.cpp @@ -0,0 +1,16 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime/memory_resource_impl/memory_resource_stats.h" + +void write_memory_stats_to(const memory_resource::MemoryStats & memoryStats, stats_t *stats, const char *prefix) noexcept { + stats->add_gauge_stat(memoryStats.memory_limit, prefix, ".memory.limit"); + stats->add_gauge_stat(memoryStats.memory_used, prefix, ".memory.used"); + stats->add_gauge_stat(memoryStats.real_memory_used, prefix, ".memory.real_used"); + stats->add_gauge_stat(memoryStats.max_memory_used, prefix, ".memory.used_max"); + stats->add_gauge_stat(memoryStats.max_real_memory_used, prefix, ".memory.real_used_max"); + stats->add_gauge_stat(memoryStats.defragmentation_calls, prefix, ".memory.defragmentation_calls"); + stats->add_gauge_stat(memoryStats.huge_memory_pieces, prefix, ".memory.huge_memory_pieces"); + stats->add_gauge_stat(memoryStats.small_memory_pieces, prefix, ".memory.small_memory_pieces"); +} \ No newline at end of file diff --git a/runtime/memory_resource_impl/memory_resource_stats.h b/runtime/memory_resource_impl/memory_resource_stats.h new file mode 100644 index 0000000000..dd41b25e12 --- /dev/null +++ b/runtime/memory_resource_impl/memory_resource_stats.h @@ -0,0 +1,10 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "common/stats/provider.h" +#include "runtime-core/memory-resource/memory_resource.h" + +void write_memory_stats_to(const memory_resource::MemoryStats & memoryStats, stats_t *stats, const char *prefix) noexcept; diff --git a/runtime/memory_resource/monotonic_buffer_resource.cpp b/runtime/memory_resource_impl/monotonic_runtime_buffer_resource.cpp similarity index 63% rename from runtime/memory_resource/monotonic_buffer_resource.cpp rename to runtime/memory_resource_impl/monotonic_runtime_buffer_resource.cpp index 2fdb5a8d82..7051dcc892 100644 --- a/runtime/memory_resource/monotonic_buffer_resource.cpp +++ b/runtime/memory_resource_impl/monotonic_runtime_buffer_resource.cpp @@ -1,26 +1,17 @@ // Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» +// Copyright (c) 2024 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt -#include "runtime/memory_resource/monotonic_buffer_resource.h" +#include "runtime-core/memory-resource/monotonic_buffer_resource.h" #include + #include "runtime/allocator.h" +#include "runtime/php_assert.h" #include "runtime/oom_handler.h" namespace memory_resource { - -void monotonic_buffer::init(void *buffer, size_t buffer_size) noexcept { - php_assert(buffer_size <= memory_buffer_limit()); - memory_begin_ = static_cast(buffer); - memory_current_ = memory_begin_; - memory_end_ = memory_begin_ + buffer_size; - - stats_ = MemoryStats{}; - stats_.memory_limit = buffer_size; -} - -void monotonic_buffer::critical_dump(void *mem, size_t size) const noexcept { +void monotonic_buffer_resource::critical_dump(void *mem, size_t size) const noexcept { std::array malloc_replacement_stacktrace_buf = {'\0'}; if (dl::is_malloc_replaced()) { const char *descr = "last_malloc_replacement_stacktrace:\n"; @@ -28,25 +19,21 @@ void monotonic_buffer::critical_dump(void *mem, size_t size) const noexcept { dl::write_last_malloc_replacement_stacktrace(malloc_replacement_stacktrace_buf.data() + strlen(descr), malloc_replacement_stacktrace_buf.size() - strlen(descr)); } - php_critical_error( - "Found unexpected memory piece:\n" - "ptr: %p\n" - "size: %zu\n" - "memory_begin: %p\n" - "memory_current: %p\n" - "memory_end: %p\n" - "memory_limit: %zu\n" - "memory_used: %zu\n" - "max_memory_used: %zu\n" - "real_memory_used: %zu\n" - "max_real_memory_used: %zu\n" - "is_malloc_replaced: %d\n" - "%s", - mem, size, memory_begin_, memory_current_, memory_end_, - stats_.memory_limit, - stats_.memory_used, stats_.max_memory_used, - stats_.real_memory_used, stats_.max_real_memory_used, - dl::is_malloc_replaced(), malloc_replacement_stacktrace_buf.data()); + php_critical_error("Found unexpected memory piece:\n" + "ptr: %p\n" + "size: %zu\n" + "memory_begin: %p\n" + "memory_current: %p\n" + "memory_end: %p\n" + "memory_limit: %zu\n" + "memory_used: %zu\n" + "max_memory_used: %zu\n" + "real_memory_used: %zu\n" + "max_real_memory_used: %zu\n" + "is_malloc_replaced: %d\n" + "%s", + mem, size, memory_begin_, memory_current_, memory_end_, stats_.memory_limit, stats_.memory_used, stats_.max_memory_used, + stats_.real_memory_used, stats_.max_real_memory_used, dl::is_malloc_replaced(), malloc_replacement_stacktrace_buf.data()); } void monotonic_buffer_resource::raise_oom(size_t size) const noexcept { @@ -78,5 +65,4 @@ void monotonic_buffer_resource::raise_oom(size_t size) const noexcept { raise(SIGUSR2); } - -} // namespace memory_resource +} \ No newline at end of file diff --git a/runtime/memory_usage.h b/runtime/memory_usage.h index c024b3ab6f..abfbf783ee 100644 --- a/runtime/memory_usage.h +++ b/runtime/memory_usage.h @@ -13,9 +13,8 @@ #include "common/mixin/not_copyable.h" #include "common/type_traits/list_of_types.h" -#include "runtime/declarations.h" -#include "runtime/kphp_core.h" -#include "runtime/shape.h" +#include "runtime-core/runtime-core.h" +#include "runtime/allocator.h" template struct CDataPtr; @@ -172,3 +171,44 @@ int64_t f$estimate_memory_usage(const T &value) { } array f$get_global_vars_memory_stats(int64_t lower_bound = 0); + +inline int64_t f$memory_get_static_usage() { + return static_cast(dl::get_heap_memory_used()); +} + +inline int64_t f$memory_get_peak_usage(bool real_usage) { + if (real_usage) { + return static_cast(dl::get_script_memory_stats().max_real_memory_used); + } else { + return static_cast(dl::get_script_memory_stats().max_memory_used); + } +} + +inline int64_t f$memory_get_usage(bool real_usage __attribute__((unused))) { + return static_cast(dl::get_script_memory_stats().memory_used); +} + +inline int64_t f$memory_get_total_usage() { + return static_cast(dl::get_script_memory_stats().real_memory_used); +} + +inline array f$memory_get_detailed_stats() { + const auto &stats = dl::get_script_memory_stats(); + return array( + { + std::make_pair(string{"memory_limit"}, static_cast(stats.memory_limit)), + std::make_pair(string{"real_memory_used"}, static_cast(stats.real_memory_used)), + std::make_pair(string{"memory_used"}, static_cast(stats.memory_used)), + std::make_pair(string{"max_real_memory_used"}, static_cast(stats.max_real_memory_used)), + std::make_pair(string{"max_memory_used"}, static_cast(stats.max_memory_used)), + std::make_pair(string{"defragmentation_calls"}, static_cast(stats.defragmentation_calls)), + std::make_pair(string{"huge_memory_pieces"}, static_cast(stats.huge_memory_pieces)), + std::make_pair(string{"small_memory_pieces"}, static_cast(stats.small_memory_pieces)), + std::make_pair(string{"heap_memory_used"}, static_cast(dl::get_heap_memory_used())) + }); +} + +inline std::tuple f$memory_get_allocations() { + const auto &stats = dl::get_script_memory_stats(); + return {stats.total_allocations, stats.total_memory_allocated}; +} diff --git a/runtime/misc.cpp b/runtime/misc.cpp index 62dfb24ead..ad82d4f385 100644 --- a/runtime/misc.cpp +++ b/runtime/misc.cpp @@ -37,18 +37,18 @@ string f$uniqid(const string &prefix, bool more_entropy) { size_t buf_size = 30; char buf[buf_size]; - static_SB.clean() << prefix; + kphp_runtime_context.static_SB.clean() << prefix; if (more_entropy) { snprintf(buf, buf_size, "%08x%05x%.8f", sec, usec, f$lcg_value() * 10); - static_SB.append(buf, 23); + kphp_runtime_context.static_SB.append(buf, 23); } else { snprintf(buf, buf_size, "%08x%05x", sec, usec); - static_SB.append(buf, 13); + kphp_runtime_context.static_SB.append(buf, 13); } dl::leave_critical_section(); - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } @@ -710,10 +710,10 @@ string f$cp1251(const string &utf8_string) { void f$kphp_set_context_on_error(const array &tags, const array &extra_info, const string& env) { auto &json_logger = vk::singleton::get(); - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); - const auto get_json_string_from_SB_without_brackets = [] { - vk::string_view json_str{static_SB.c_str(), static_SB.size()}; + const auto get_json_string_from_SB_without_brackets = [&] { + vk::string_view json_str{kphp_runtime_context.static_SB.c_str(), kphp_runtime_context.static_SB.size()}; php_assert(json_str.size() >= 2 && json_str.front() == '{' && json_str.back() == '}'); json_str.remove_prefix(1); json_str.remove_suffix(1); @@ -725,14 +725,14 @@ void f$kphp_set_context_on_error(const array &tags, const array &e dl::CriticalSectionGuard critical_section; json_logger.set_tags(tags_json); } - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); if (impl_::JsonEncoder(JSON_FORCE_OBJECT, false).encode(extra_info)) { auto extra_info_json = get_json_string_from_SB_without_brackets(); dl::CriticalSectionGuard critical_section; json_logger.set_extra_info(extra_info_json); } - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); dl::CriticalSectionGuard critical_section; json_logger.set_env({env.c_str(), env.size()}); diff --git a/runtime/misc.h b/runtime/misc.h index 50cd617e56..cb7e969bd8 100644 --- a/runtime/misc.h +++ b/runtime/misc.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" string f$uniqid(const string &prefix = string(), bool more_entropy = false); diff --git a/runtime/msgpack-serialization.h b/runtime/msgpack-serialization.h index e0b594d0a0..37a1f2fee2 100644 --- a/runtime/msgpack-serialization.h +++ b/runtime/msgpack-serialization.h @@ -11,10 +11,11 @@ #include "common/containers/final_action.h" +#include "runtime-core/runtime-core.h" +#include "runtime/context/runtime-context.h" #include "runtime/critical_section.h" #include "runtime/exception.h" #include "runtime/interface.h" -#include "runtime/kphp_core.h" #include "runtime/string_functions.h" template @@ -22,15 +23,15 @@ inline Optional f$msgpack_serialize(const T &value, string *out_err_msg f$ob_start(); php_assert(f$ob_get_length().has_value() && f$ob_get_length().val() == 0); - string_buffer::string_buffer_error_flag = STRING_BUFFER_ERROR_FLAG_ON; + kphp_runtime_context.sb_lib_context.error_flag = STRING_BUFFER_ERROR_FLAG_ON; auto clean_buffer = vk::finally([] { f$ob_end_clean(); - string_buffer::string_buffer_error_flag = STRING_BUFFER_ERROR_FLAG_OFF; + kphp_runtime_context.sb_lib_context.error_flag = STRING_BUFFER_ERROR_FLAG_OFF; }); vk::msgpack::packer{*coub}.pack(value); - if (string_buffer::string_buffer_error_flag == STRING_BUFFER_ERROR_FLAG_FAILED) { + if (kphp_runtime_context.sb_lib_context.error_flag == STRING_BUFFER_ERROR_FLAG_FAILED) { string err_msg{"msgpacke_serialize buffer overflow"}; if (out_err_msg) { *out_err_msg = std::move(err_msg); diff --git a/runtime/msgpack/adaptors.h b/runtime/msgpack/adaptors.h index 7de7e26cf6..fa2070635b 100644 --- a/runtime/msgpack/adaptors.h +++ b/runtime/msgpack/adaptors.h @@ -7,7 +7,7 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/msgpack/adaptor_base.h" #include "runtime/msgpack/check_instance_depth.h" #include "runtime/msgpack/object.h" diff --git a/runtime/msgpack/packer.cpp b/runtime/msgpack/packer.cpp index 28eb12c03b..914bd10556 100644 --- a/runtime/msgpack/packer.cpp +++ b/runtime/msgpack/packer.cpp @@ -5,7 +5,7 @@ #include "runtime/msgpack/packer.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/msgpack/sysdep.h" namespace vk::msgpack { diff --git a/runtime/msgpack/unpacker.h b/runtime/msgpack/unpacker.h index 7bd5471db2..6784ae215e 100644 --- a/runtime/msgpack/unpacker.h +++ b/runtime/msgpack/unpacker.h @@ -6,7 +6,7 @@ #pragma once #include "common/mixin/not_copyable.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/msgpack/object.h" #include "runtime/msgpack/zone.h" diff --git a/runtime/mysql.h b/runtime/mysql.h index 9c001fd53b..4d812ced0f 100644 --- a/runtime/mysql.h +++ b/runtime/mysql.h @@ -4,10 +4,10 @@ #pragma once +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/kphp_core.h" #include "runtime/memory_usage.h" -#include "runtime/refcountable_php_classes.h" class C$mysqli : public refcountable_php_classes, private DummyVisitorMethods { public: diff --git a/runtime/null_coalesce.h b/runtime/null_coalesce.h index 159eeb35d9..146c4e0ff3 100644 --- a/runtime/null_coalesce.h +++ b/runtime/null_coalesce.h @@ -5,8 +5,8 @@ #pragma once #include -#include "runtime/include.h" -#include "runtime/kphp_core.h" +#include "runtime-core/include.h" +#include "runtime-core/runtime-core.h" namespace impl_ { diff --git a/runtime/on_kphp_warning_callback.h b/runtime/on_kphp_warning_callback.h index 7b48bcf81b..6896c8f816 100644 --- a/runtime/on_kphp_warning_callback.h +++ b/runtime/on_kphp_warning_callback.h @@ -5,8 +5,8 @@ #pragma once #include +#include "runtime-core/runtime-core.h" #include "runtime/critical_section.h" -#include "runtime/kphp_core.h" using on_kphp_warning_callback_type = std::function &)>; diff --git a/runtime/openssl.cpp b/runtime/openssl.cpp index cf7c91433b..ef6635596b 100644 --- a/runtime/openssl.cpp +++ b/runtime/openssl.cpp @@ -29,6 +29,7 @@ #include "common/wrappers/to_array.h" #include "runtime/array_functions.h" +#include "runtime/allocator.h" #include "runtime/critical_section.h" #include "runtime/datetime/datetime_functions.h" #include "runtime/files.h" @@ -515,11 +516,11 @@ static const EVP_MD *openssl_algo_to_evp_md(openssl_algo algo) { } static const char *ssl_get_error_string() { - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); while (unsigned long error_code = ERR_get_error()) { - static_SB << "Error " << (int)error_code << ": [" << ERR_error_string(error_code, nullptr) << "]\n"; + kphp_runtime_context.static_SB << "Error " << (int)error_code << ": [" << ERR_error_string(error_code, nullptr) << "]\n"; } - return static_SB.c_str(); + return kphp_runtime_context.static_SB.c_str(); } bool f$openssl_sign(const string &data, string &signature, const string &priv_key_id, int64_t algo) { diff --git a/runtime/openssl.h b/runtime/openssl.h index 06ba6d70be..21ffb44d79 100644 --- a/runtime/openssl.h +++ b/runtime/openssl.h @@ -6,7 +6,7 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" enum openssl_algo { OPENSSL_ALGO_SHA1 = 1, diff --git a/runtime/pdo/abstract_pdo_driver.h b/runtime/pdo/abstract_pdo_driver.h index 74e318e148..7c574559da 100644 --- a/runtime/pdo/abstract_pdo_driver.h +++ b/runtime/pdo/abstract_pdo_driver.h @@ -6,8 +6,8 @@ #include +#include "runtime-core/runtime-core.h" #include "runtime/allocator.h" -#include "runtime/kphp_core.h" struct C$PDO; struct C$PDOStatement; diff --git a/runtime/pdo/abstract_pdo_statement.h b/runtime/pdo/abstract_pdo_statement.h index e503e63011..26737846dc 100644 --- a/runtime/pdo/abstract_pdo_statement.h +++ b/runtime/pdo/abstract_pdo_statement.h @@ -4,8 +4,8 @@ #pragma once +#include "runtime-core/runtime-core.h" #include "runtime/allocator.h" -#include "runtime/kphp_core.h" struct C$PDO; struct C$PDOStatement; diff --git a/runtime/pdo/mysql/mysql_pdo_driver.h b/runtime/pdo/mysql/mysql_pdo_driver.h index 231fb978af..f362a69b9e 100644 --- a/runtime/pdo/mysql/mysql_pdo_driver.h +++ b/runtime/pdo/mysql/mysql_pdo_driver.h @@ -4,8 +4,8 @@ #pragma once +#include "runtime-core/runtime-core.h" #include "runtime/allocator.h" -#include "runtime/kphp_core.h" #include "runtime/pdo/abstract_pdo_driver.h" namespace pdo::mysql { diff --git a/runtime/pdo/mysql/mysql_pdo_emulated_statement.h b/runtime/pdo/mysql/mysql_pdo_emulated_statement.h index 66c3f78629..b15121d80f 100644 --- a/runtime/pdo/mysql/mysql_pdo_emulated_statement.h +++ b/runtime/pdo/mysql/mysql_pdo_emulated_statement.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/pdo/abstract_pdo_statement.h" namespace database_drivers { diff --git a/runtime/pdo/pdo.h b/runtime/pdo/pdo.h index f22a3a7f5e..46b9f0c022 100644 --- a/runtime/pdo/pdo.h +++ b/runtime/pdo/pdo.h @@ -9,11 +9,11 @@ #include "common/algorithms/hashes.h" #include "common/wrappers/string_view.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/kphp_core.h" #include "runtime/memory_usage.h" #include "runtime/pdo/abstract_pdo_driver.h" -#include "runtime/refcountable_php_classes.h" struct C$PDO : public refcountable_polymorphic_php_classes, private DummyVisitorMethods { static constexpr int ATTR_TIMEOUT = 2; diff --git a/runtime/pdo/pdo_statement.h b/runtime/pdo/pdo_statement.h index 17aa965d3a..3243117deb 100644 --- a/runtime/pdo/pdo_statement.h +++ b/runtime/pdo/pdo_statement.h @@ -9,13 +9,12 @@ #include "common/algorithms/hashes.h" #include "common/wrappers/string_view.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/kphp_core.h" #include "runtime/memory_usage.h" -#include "runtime/refcountable_php_classes.h" #include "runtime/pdo/abstract_pdo_statement.h" - struct C$PDOStatement : public refcountable_polymorphic_php_classes, private DummyVisitorMethods { std::unique_ptr statement; int64_t timeout_sec{-1}; diff --git a/runtime/pdo/pgsql/pgsql_pdo_driver.h b/runtime/pdo/pgsql/pgsql_pdo_driver.h index bcc4d577b1..8b7d22b0b1 100644 --- a/runtime/pdo/pgsql/pgsql_pdo_driver.h +++ b/runtime/pdo/pgsql/pgsql_pdo_driver.h @@ -1,7 +1,7 @@ #pragma once +#include "runtime-core/runtime-core.h" #include "runtime/allocator.h" -#include "runtime/kphp_core.h" #include "runtime/pdo/abstract_pdo_driver.h" namespace pdo::pgsql { diff --git a/runtime/pdo/pgsql/pgsql_pdo_emulated_statement.cpp b/runtime/pdo/pgsql/pgsql_pdo_emulated_statement.cpp index 5ef7e6f47c..46e0a32e7d 100644 --- a/runtime/pdo/pgsql/pgsql_pdo_emulated_statement.cpp +++ b/runtime/pdo/pgsql/pgsql_pdo_emulated_statement.cpp @@ -1,8 +1,8 @@ #include -#include "runtime/pdo/pgsql/pgsql_pdo_emulated_statement.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/pdo/pdo_statement.h" +#include "runtime/pdo/pgsql/pgsql_pdo_emulated_statement.h" #include "runtime/resumable.h" #include "server/database-drivers/adaptor.h" #include "server/database-drivers/pgsql/pgsql-request.h" diff --git a/runtime/pdo/pgsql/pgsql_pdo_emulated_statement.h b/runtime/pdo/pgsql/pgsql_pdo_emulated_statement.h index 9f578751dc..ed130ae27b 100644 --- a/runtime/pdo/pgsql/pgsql_pdo_emulated_statement.h +++ b/runtime/pdo/pgsql/pgsql_pdo_emulated_statement.h @@ -1,6 +1,6 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/pdo/abstract_pdo_statement.h" namespace database_drivers { diff --git a/runtime/php-script-globals.h b/runtime/php-script-globals.h index 028a165c56..a81bd03d64 100644 --- a/runtime/php-script-globals.h +++ b/runtime/php-script-globals.h @@ -6,7 +6,7 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" struct PhpScriptBuiltInSuperGlobals { // variables below are PHP language superglobals diff --git a/runtime/php_assert.cpp b/runtime/php_assert.cpp index c8f11fc95b..5b3b8a5d57 100644 --- a/runtime/php_assert.cpp +++ b/runtime/php_assert.cpp @@ -236,4 +236,20 @@ void php_assert__(const char *msg, const char *file, int line) { void raise_php_assert_signal__() { raise(SIGPHPASSERT); vk::singleton::get().fsync_log_file(); + _exit(1); +} + +int64_t f$error_reporting(int64_t level) { + int32_t prev = php_warning_level; + if ((level & E_ALL) == E_ALL) { + php_warning_level = 3; + } + if (0 <= level && level <= 3) { + php_warning_level = std::max(php_warning_minimum_level, static_cast(level)); + } + return prev; +} + +int64_t f$error_reporting() { + return php_warning_level; } diff --git a/runtime/php_assert.h b/runtime/php_assert.h index c4514f9b2f..0532199bac 100644 --- a/runtime/php_assert.h +++ b/runtime/php_assert.h @@ -6,10 +6,13 @@ #include #include +#include #include "common/wrappers/likely.h" #include "common/mixin/not_copyable.h" +#include "runtime-core/utils/kphp-assert-core.h" + extern int die_on_fail; extern const char *engine_tag; @@ -19,9 +22,6 @@ extern int php_disable_warnings; extern int php_warning_level; extern int php_warning_minimum_level; -void php_notice(char const *message, ...) __attribute__ ((format (printf, 1, 2))); -void php_warning(char const *message, ...) __attribute__ ((format (printf, 1, 2))); -void php_error(char const *message, ...) __attribute__ ((format (printf, 1, 2))); void php_out_of_memory_warning(char const *message, ...) __attribute__ ((format (printf, 1, 2))); template @@ -29,18 +29,6 @@ class class_instance; struct C$Throwable; const char *php_uncaught_exception_error(const class_instance &ex) noexcept; -void php_assert__(const char *msg, const char *file, int line) __attribute__((noreturn)); -void raise_php_assert_signal__(); - -#define php_assert(EX) do { \ - if (unlikely(!(EX))) { \ - php_assert__ (#EX, __FILE__, __LINE__); \ - } \ -} while(0) - -#define php_critical_error(format, ...) do { \ - php_error ("Critical error \"" format "\" in file %s on line %d", ##__VA_ARGS__, __FILE__, __LINE__); \ - raise_php_assert_signal__(); \ - fprintf (stderr, "_exiting in php_critical_error\n"); \ - _exit (1); \ -} while(0) +int64_t f$error_reporting(int64_t level); + +int64_t f$error_reporting(); diff --git a/runtime/profiler.h b/runtime/profiler.h index 742ab2706c..cb5ae96270 100644 --- a/runtime/profiler.h +++ b/runtime/profiler.h @@ -12,7 +12,8 @@ #include "common/mixin/not_copyable.h" #include "common/wrappers/string_view.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" +#include "runtime/allocator.h" #include "server/php-queries-stats.h" template diff --git a/runtime/regexp.cpp b/runtime/regexp.cpp index 16669bf205..3fb1a67db7 100644 --- a/runtime/regexp.cpp +++ b/runtime/regexp.cpp @@ -412,7 +412,7 @@ void regexp::init(const char *regexp_string, int64_t regexp_len, const char *fun return; } - static_SB.clean().append(regexp_string + 1, static_cast(regexp_end - 1)); + kphp_runtime_context.static_SB.clean().append(regexp_string + 1, static_cast(regexp_end - 1)); use_heap_memory = !(php_script.has_value() && php_script->is_running()); @@ -478,23 +478,22 @@ void regexp::init(const char *regexp_string, int64_t regexp_len, const char *fun } } - can_use_RE2 = can_use_RE2 && is_valid_RE2_regexp(static_SB.c_str(), static_SB.size(), is_utf8, function, file); + can_use_RE2 = can_use_RE2 && is_valid_RE2_regexp(kphp_runtime_context.static_SB.c_str(), kphp_runtime_context.static_SB.size(), is_utf8, function, file); - if (is_utf8 && !mb_UTF8_check(static_SB.c_str())) { - pattern_compilation_warning(function, file, "Regexp \"%s\" contains not UTF-8 symbols", static_SB.c_str()); + if (is_utf8 && !mb_UTF8_check(kphp_runtime_context.static_SB.c_str())) { + pattern_compilation_warning(function, file, "Regexp \"%s\" contains not UTF-8 symbols", kphp_runtime_context.static_SB.c_str()); clean(); return; } bool need_pcre = false; if (can_use_RE2) { - RE2_regexp = new RE2(re2::StringPiece(static_SB.c_str(), static_SB.size()), RE2_options); + RE2_regexp = new RE2(re2::StringPiece(kphp_runtime_context.static_SB.c_str(), kphp_runtime_context.static_SB.size()), RE2_options); #if ASAN_ENABLED __lsan_ignore_object(RE2_regexp); #endif if (!RE2_regexp->ok()) { - pattern_compilation_warning(function, file, "RE2 compilation of regexp \"%s\" failed. Error %d at %s", - static_SB.c_str(), RE2_regexp->error_code(), RE2_regexp->error().c_str()); + pattern_compilation_warning(function, file, "RE2 compilation of regexp \"%s\" failed. Error %d at %s", kphp_runtime_context.static_SB.c_str(), RE2_regexp->error_code(), RE2_regexp->error().c_str()); delete RE2_regexp; RE2_regexp = nullptr; @@ -514,7 +513,7 @@ void regexp::init(const char *regexp_string, int64_t regexp_len, const char *fun if (RE2_regexp == nullptr || need_pcre) { const char *error; int32_t erroffset = 0; - pcre_regexp = pcre_compile(static_SB.c_str(), pcre_options, &error, &erroffset, nullptr); + pcre_regexp = pcre_compile(kphp_runtime_context.static_SB.c_str(), pcre_options, &error, &erroffset, nullptr); #if ASAN_ENABLED __lsan_ignore_object(pcre_regexp); #endif @@ -1078,7 +1077,7 @@ int64_t regexp::last_error() { string f$preg_quote(const string &str, const string &delimiter) { const string::size_type len = str.size(); - static_SB.clean().reserve(4 * len); + kphp_runtime_context.static_SB.clean().reserve(4 * len); for (string::size_type i = 0; i < len; i++) { switch (str[i]) { @@ -1103,25 +1102,25 @@ string f$preg_quote(const string &str, const string &delimiter) { case ':': case '-': case '#': - static_SB.append_char('\\'); - static_SB.append_char(str[i]); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char(str[i]); break; case '\0': - static_SB.append_char('\\'); - static_SB.append_char('0'); - static_SB.append_char('0'); - static_SB.append_char('0'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('0'); + kphp_runtime_context.static_SB.append_char('0'); + kphp_runtime_context.static_SB.append_char('0'); break; default: if (!delimiter.empty() && str[i] == delimiter[0]) { - static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('\\'); } - static_SB.append_char(str[i]); + kphp_runtime_context.static_SB.append_char(str[i]); break; } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } void regexp::global_init() { diff --git a/runtime/regexp.h b/runtime/regexp.h index 40fbde3308..1de0260cee 100644 --- a/runtime/regexp.h +++ b/runtime/regexp.h @@ -8,9 +8,10 @@ #include "common/mixin/not_copyable.h" -#include "runtime/kphp_core.h" -#include "runtime/mbstring.h" +#include "runtime-core/runtime-core.h" +#include "runtime/context/runtime-context.h" #include "runtime/kphp_tracing.h" +#include "runtime/mbstring.h" namespace re2 { class RE2; @@ -210,8 +211,7 @@ inline int64_t f$preg_last_error(); template<> inline string regexp::get_replacement(const string &replace_val, const string &subject, int64_t count) const { const string::size_type len = replace_val.size(); - - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); for (string::size_type i = 0; i < len; i++) { int64_t backref = -1; if (replace_val[i] == '\\' && (replace_val[i + 1] == '\\' || replace_val[i + 1] == '$')) { @@ -239,16 +239,16 @@ inline string regexp::get_replacement(const string &replace_val, const string &s } if (backref == -1) { - static_SB << replace_val[i]; + kphp_runtime_context.static_SB << replace_val[i]; } else { if (backref < count) { int64_t index = backref + backref; - static_SB.append(subject.c_str() + submatch[index], + kphp_runtime_context.static_SB.append(subject.c_str() + submatch[index], static_cast(submatch[index + 1] - submatch[index])); } } } - return static_SB.str();//TODO optimize + return kphp_runtime_context.static_SB.str();//TODO optimize } template diff --git a/runtime/resumable.h b/runtime/resumable.h index 56df39a775..649e40aaf0 100644 --- a/runtime/resumable.h +++ b/runtime/resumable.h @@ -4,9 +4,9 @@ #pragma once +#include "runtime-core/runtime-core.h" #include "runtime/allocator.h" #include "runtime/exception.h" -#include "runtime/kphp_core.h" #include "runtime/storage.h" extern bool resumable_finished; diff --git a/runtime/rpc.cpp b/runtime/rpc.cpp index d476cc54e0..0568326b81 100644 --- a/runtime/rpc.cpp +++ b/runtime/rpc.cpp @@ -12,6 +12,7 @@ #include "common/rpc-headers.h" #include "common/tl/constants/common.h" +#include "runtime/context/runtime-context.h" #include "runtime/critical_section.h" #include "runtime/exception.h" #include "runtime/kphp_tracing.h" @@ -1395,9 +1396,9 @@ class rpc_tl_query_result_resumable : public Resumable { int64_t query_id = it.get_value(); if (!tl_objects_unsorted.isset(query_id)) { if (query_id <= 0) { - tl_objects[it.get_key()] = tl_fetch_error((static_SB.clean() << "Very wrong query_id " << query_id).str(), TL_ERROR_WRONG_QUERY_ID); + tl_objects[it.get_key()] = tl_fetch_error((kphp_runtime_context.static_SB.clean() << "Very wrong query_id " << query_id).str(), TL_ERROR_WRONG_QUERY_ID); } else { - tl_objects[it.get_key()] = tl_fetch_error((static_SB.clean() << "No answer received or duplicate/wrong query_id " + tl_objects[it.get_key()] = tl_fetch_error((kphp_runtime_context.static_SB.clean() << "No answer received or duplicate/wrong query_id " << query_id).str(), TL_ERROR_WRONG_QUERY_ID); } } else { @@ -1448,9 +1449,9 @@ array> f$rpc_tl_query_result_synchronously(const array &qu int64_t query_id = it.get_value(); if (!tl_objects_unsorted.isset(query_id)) { if (query_id <= 0) { - tl_objects[it.get_key()] = tl_fetch_error((static_SB.clean() << "Very wrong query_id " << query_id).str(), TL_ERROR_WRONG_QUERY_ID); + tl_objects[it.get_key()] = tl_fetch_error((kphp_runtime_context.static_SB.clean() << "Very wrong query_id " << query_id).str(), TL_ERROR_WRONG_QUERY_ID); } else { - tl_objects[it.get_key()] = tl_fetch_error((static_SB.clean() << "No answer received or duplicate/wrong query_id " + tl_objects[it.get_key()] = tl_fetch_error((kphp_runtime_context.static_SB.clean() << "No answer received or duplicate/wrong query_id " << query_id).str(), TL_ERROR_WRONG_QUERY_ID); } } else { diff --git a/runtime/rpc.h b/runtime/rpc.h index 7cf9844758..696ae41dc1 100644 --- a/runtime/rpc.h +++ b/runtime/rpc.h @@ -8,12 +8,12 @@ #include "common/algorithms/hashes.h" #include "common/kprintf.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/kphp_core.h" #include "runtime/net_events.h" #include "runtime/resumable.h" -#include "runtime/to-array-processor.h" #include "runtime/rpc_extra_info.h" +#include "runtime/to-array-processor.h" DECLARE_VERBOSITY(rpc); diff --git a/runtime/rpc_extra_info.h b/runtime/rpc_extra_info.h index db55ca3bad..7de8d95af8 100644 --- a/runtime/rpc_extra_info.h +++ b/runtime/rpc_extra_info.h @@ -7,12 +7,11 @@ #include #include -#include "runtime/kphp_core.h" -#include "runtime/dummy-visitor-methods.h" -#include "runtime/refcountable_php_classes.h" #include "common/algorithms/hashes.h" #include "common/wrappers/string_view.h" - +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" +#include "runtime/dummy-visitor-methods.h" using rpc_request_extra_info_t = std::tuple; // tuple(request_size) using rpc_response_extra_info_t = std::tuple; // tuple(response_size, response_time) diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 62215b7066..0ca8ebacf1 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -6,14 +6,11 @@ prepend(KPHP_RUNTIME_DATETIME_SOURCES datetime/ datetime_zone.cpp timelib_wrapper.cpp) -prepend(KPHP_RUNTIME_MEMORY_RESOURCE_SOURCES memory_resource/ +prepend(KPHP_RUNTIME_MEMORY_IMPL_RESOURCE_SOURCES memory_resource_impl/ dealer.cpp - details/memory_chunk_tree.cpp - details/memory_ordered_chunk_list.cpp heap_resource.cpp - memory_resource.cpp - monotonic_buffer_resource.cpp - unsynchronized_pool_resource.cpp) + memory_resource_stats.cpp + monotonic_runtime_buffer_resource.cpp) prepend(KPHP_RUNTIME_MSGPACK_SOURCES msgpack/ check_instance_depth.cpp @@ -59,7 +56,7 @@ endif() prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ ${KPHP_RUNTIME_DATETIME_SOURCES} - ${KPHP_RUNTIME_MEMORY_RESOURCE_SOURCES} + ${KPHP_RUNTIME_MEMORY_IMPL_RESOURCE_SOURCES} ${KPHP_RUNTIME_MSGPACK_SOURCES} ${KPHP_RUNTIME_JOB_WORKERS_SOURCES} ${KPHP_RUNTIME_SPL_SOURCES} @@ -68,6 +65,9 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ ${KPHP_RUNTIME_PDO_MYSQL_SOURCES} ${KPHP_RUNTIME_PDO_PGSQL_SOURCES} allocator.cpp + context/runtime-core-allocator.cpp + context/runtime-core-context.cpp + context/runtime-context.cpp array_functions.cpp bcmath.cpp common_template_instantiations.cpp @@ -97,9 +97,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ mbstring.cpp memcache.cpp memory_usage.cpp - migration_php8.cpp misc.cpp - mixed.cpp mysql.cpp net_events.cpp on_kphp_warning_callback.cpp @@ -115,9 +113,6 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ serialize-functions.cpp storage.cpp streams.cpp - string.cpp - string_buffer.cpp - string_cache.cpp string_functions.cpp tl/rpc_req_error.cpp tl/rpc_tl_query.cpp @@ -159,7 +154,7 @@ target_include_directories(kphp_runtime PUBLIC ${BASE_DIR} /opt/curl7600/include add_dependencies(kphp_runtime kphp-timelib) prepare_cross_platform_libs(RUNTIME_LIBS yaml-cpp re2 zstd h3) # todo: linking between static libs is no-op, is this redundant? do we need to add mysqlclient here? -set(RUNTIME_LIBS vk::kphp_runtime vk::kphp_server vk::popular_common vk::unicode vk::common_src vk::binlog_src vk::net_src ${RUNTIME_LIBS} OpenSSL::Crypto m z pthread) +set(RUNTIME_LIBS vk::kphp_runtime vk::kphp_server vk::runtime-core vk::popular_common vk::unicode vk::common_src vk::binlog_src vk::net_src ${RUNTIME_LIBS} OpenSSL::Crypto m z pthread) vk_add_library(kphp-full-runtime STATIC) target_link_libraries(kphp-full-runtime PUBLIC ${RUNTIME_LIBS}) set_target_properties(kphp-full-runtime PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR}) diff --git a/runtime/serialize-functions.cpp b/runtime/serialize-functions.cpp index 736a6c4580..ba7cadd15b 100644 --- a/runtime/serialize-functions.cpp +++ b/runtime/serialize-functions.cpp @@ -4,44 +4,46 @@ #include "runtime/serialize-functions.h" +#include "runtime/context/runtime-context.h" + void impl_::PhpSerializer::serialize(bool b) noexcept { - static_SB.reserve(4); - static_SB.append_char('b'); - static_SB.append_char(':'); - static_SB.append_char(static_cast(b + '0')); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.reserve(4); + kphp_runtime_context.static_SB.append_char('b'); + kphp_runtime_context.static_SB.append_char(':'); + kphp_runtime_context.static_SB.append_char(static_cast(b + '0')); + kphp_runtime_context.static_SB.append_char(';'); } void impl_::PhpSerializer::serialize(int64_t i) noexcept { - static_SB.reserve(24); - static_SB.append_char('i'); - static_SB.append_char(':'); - static_SB << i; - static_SB.append_char(';'); + kphp_runtime_context.static_SB.reserve(24); + kphp_runtime_context.static_SB.append_char('i'); + kphp_runtime_context.static_SB.append_char(':'); + kphp_runtime_context.static_SB << i; + kphp_runtime_context.static_SB.append_char(';'); } void impl_::PhpSerializer::serialize(double f) noexcept { - static_SB.append("d:", 2); - static_SB << f << ';'; + kphp_runtime_context.static_SB.append("d:", 2); + kphp_runtime_context.static_SB << f << ';'; } void impl_::PhpSerializer::serialize(const string &s) noexcept { string::size_type len = s.size(); - static_SB.reserve(25 + len); - static_SB.append_char('s'); - static_SB.append_char(':'); - static_SB << len; - static_SB.append_char(':'); - static_SB.append_char('"'); - static_SB.append_unsafe(s.c_str(), len); - static_SB.append_char('"'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.reserve(25 + len); + kphp_runtime_context.static_SB.append_char('s'); + kphp_runtime_context.static_SB.append_char(':'); + kphp_runtime_context.static_SB << len; + kphp_runtime_context.static_SB.append_char(':'); + kphp_runtime_context.static_SB.append_char('"'); + kphp_runtime_context.static_SB.append_unsafe(s.c_str(), len); + kphp_runtime_context.static_SB.append_char('"'); + kphp_runtime_context.static_SB.append_char(';'); } void impl_::PhpSerializer::serialize_null() noexcept { - static_SB.reserve(2); - static_SB.append_char('N'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.reserve(2); + kphp_runtime_context.static_SB.append_char('N'); + kphp_runtime_context.static_SB.append_char(';'); } void impl_::PhpSerializer::serialize(const mixed &v) noexcept { diff --git a/runtime/serialize-functions.h b/runtime/serialize-functions.h index c3bdd15f84..9ebb60a5ce 100644 --- a/runtime/serialize-functions.h +++ b/runtime/serialize-functions.h @@ -4,7 +4,9 @@ #pragma once -#include "runtime/kphp_core.h" +#include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" +#include "runtime/context/runtime-context.h" namespace impl_ { @@ -28,9 +30,9 @@ class PhpSerializer : vk::not_copyable { template void PhpSerializer::serialize(const array &arr) noexcept { - static_SB.append("a:", 2); - static_SB << arr.count(); - static_SB.append(":{", 2); + kphp_runtime_context.static_SB.append("a:", 2); + kphp_runtime_context.static_SB << arr.count(); + kphp_runtime_context.static_SB.append(":{", 2); for (auto p : arr) { auto key = p.get_key(); if (array::is_int_key(key)) { @@ -40,7 +42,7 @@ void PhpSerializer::serialize(const array &arr) noexcept { } serialize(p.get_value()); } - static_SB << '}'; + kphp_runtime_context.static_SB << '}'; } template @@ -59,9 +61,9 @@ void PhpSerializer::serialize(const Optional &opt) noexcept { template string f$serialize(const T &v) noexcept { - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); impl_::PhpSerializer::serialize(v); - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } mixed f$unserialize(const string &v) noexcept; diff --git a/runtime/spl/array_iterator.h b/runtime/spl/array_iterator.h index 76ad9aabf7..07d023a680 100644 --- a/runtime/spl/array_iterator.h +++ b/runtime/spl/array_iterator.h @@ -7,9 +7,9 @@ #include "common/algorithms/hashes.h" #include "common/wrappers/string_view.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" -#include "runtime/kphp_core.h" -#include "runtime/refcountable_php_classes.h" // C$ArrayIterator implements SPL ArrayIterator class. struct C$ArrayIterator : public refcountable_php_classes, private DummyVisitorMethods { diff --git a/runtime/storage.h b/runtime/storage.h index 870fa81d80..b778b8b75b 100644 --- a/runtime/storage.h +++ b/runtime/storage.h @@ -6,8 +6,8 @@ #include +#include "runtime-core/runtime-core.h" #include "runtime/exception.h" -#include "runtime/kphp_core.h" extern const char *last_wait_error; diff --git a/runtime/streams.cpp b/runtime/streams.cpp index 77f6901850..f6a5a2e7e5 100644 --- a/runtime/streams.cpp +++ b/runtime/streams.cpp @@ -9,6 +9,7 @@ #include #include "runtime/array_functions.h" +#include "runtime/allocator.h" #include "runtime/critical_section.h" constexpr int PHP_CSV_NO_ESCAPE = EOF; diff --git a/runtime/streams.h b/runtime/streams.h index b95321e5b5..1f1ccc6b6f 100644 --- a/runtime/streams.h +++ b/runtime/streams.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" using Stream =mixed; diff --git a/runtime/string-list.h b/runtime/string-list.h index 8adc349638..f7af13ce1a 100644 --- a/runtime/string-list.h +++ b/runtime/string-list.h @@ -4,8 +4,9 @@ #pragma once +#include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" #include "runtime/allocator.h" -#include "runtime/kphp_core.h" class string_list : vk::not_copyable { public: diff --git a/runtime/string_buffer.cpp b/runtime/string_buffer.cpp deleted file mode 100644 index ff62489b97..0000000000 --- a/runtime/string_buffer.cpp +++ /dev/null @@ -1,24 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "runtime/allocator.h" -#include "runtime/kphp_core.h" - -string_buffer static_SB; -string_buffer static_SB_spare; - -string::size_type string_buffer::MIN_BUFFER_LEN = 266175; -string::size_type string_buffer::MAX_BUFFER_LEN = (1 << 24); - -int string_buffer::string_buffer_error_flag = 0; - -string_buffer::string_buffer(string::size_type buffer_len) noexcept: - buffer_end(static_cast(dl::heap_allocate(buffer_len))), - buffer_begin(buffer_end), - buffer_len(buffer_len) { -} - -string_buffer::~string_buffer() noexcept { - dl::heap_deallocate(buffer_begin, buffer_len); -} diff --git a/runtime/string_functions.cpp b/runtime/string_functions.cpp index a467e36d29..d61931bec0 100644 --- a/runtime/string_functions.cpp +++ b/runtime/string_functions.cpp @@ -11,6 +11,7 @@ #include "common/macos-ports.h" #include "common/unicode/unicode-utils.h" +#include "runtime/context/runtime-context.h" #include "runtime/interface.h" const string COLON(",", 1); @@ -54,70 +55,70 @@ string f$addcslashes(const string &str, const string &what) { const char *mask = get_mask(what); int len = str.size(); - static_SB.clean().reserve(4 * len); + kphp_runtime_context.static_SB.clean().reserve(4 * len); for (int i = 0; i < len; i++) { unsigned char c = str[i]; if (mask[c]) { - static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('\\'); if (c < 32 || c > 126) { switch (c) { case '\n': - static_SB.append_char('n'); + kphp_runtime_context.static_SB.append_char('n'); break; case '\t': - static_SB.append_char('t'); + kphp_runtime_context.static_SB.append_char('t'); break; case '\r': - static_SB.append_char('r'); + kphp_runtime_context.static_SB.append_char('r'); break; case '\a': - static_SB.append_char('a'); + kphp_runtime_context.static_SB.append_char('a'); break; case '\v': - static_SB.append_char('v'); + kphp_runtime_context.static_SB.append_char('v'); break; case '\b': - static_SB.append_char('b'); + kphp_runtime_context.static_SB.append_char('b'); break; case '\f': - static_SB.append_char('f'); + kphp_runtime_context.static_SB.append_char('f'); break; default: - static_SB.append_char(static_cast((c >> 6) + '0')); - static_SB.append_char(static_cast(((c >> 3) & 7) + '0')); - static_SB.append_char(static_cast((c & 7) + '0')); + kphp_runtime_context.static_SB.append_char(static_cast((c >> 6) + '0')); + kphp_runtime_context.static_SB.append_char(static_cast(((c >> 3) & 7) + '0')); + kphp_runtime_context.static_SB.append_char(static_cast((c & 7) + '0')); } } else { - static_SB.append_char(c); + kphp_runtime_context.static_SB.append_char(c); } } else { - static_SB.append_char(c); + kphp_runtime_context.static_SB.append_char(c); } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } string f$addslashes(const string &str) { int len = str.size(); - static_SB.clean().reserve(2 * len); + kphp_runtime_context.static_SB.clean().reserve(2 * len); for (int i = 0; i < len; i++) { switch (str[i]) { case '\0': - static_SB.append_char('\\'); - static_SB.append_char('0'); + kphp_runtime_context.static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('0'); break; case '\'': case '\"': case '\\': - static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('\\'); /* fallthrough */ default: - static_SB.append_char(str[i]); + kphp_runtime_context.static_SB.append_char(str[i]); } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } string f$bin2hex(const string &str) { @@ -329,48 +330,48 @@ static const char *cp1251_to_utf8_str[128] = { string f$htmlentities(const string &str) { int len = (int)str.size(); - static_SB.clean().reserve(8 * len); + kphp_runtime_context.static_SB.clean().reserve(8 * len); for (int i = 0; i < len; i++) { switch (str[i]) { case '&': - static_SB.append_char('&'); - static_SB.append_char('a'); - static_SB.append_char('m'); - static_SB.append_char('p'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.append_char('&'); + kphp_runtime_context.static_SB.append_char('a'); + kphp_runtime_context.static_SB.append_char('m'); + kphp_runtime_context.static_SB.append_char('p'); + kphp_runtime_context.static_SB.append_char(';'); break; case '"': - static_SB.append_char('&'); - static_SB.append_char('q'); - static_SB.append_char('u'); - static_SB.append_char('o'); - static_SB.append_char('t'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.append_char('&'); + kphp_runtime_context.static_SB.append_char('q'); + kphp_runtime_context.static_SB.append_char('u'); + kphp_runtime_context.static_SB.append_char('o'); + kphp_runtime_context.static_SB.append_char('t'); + kphp_runtime_context.static_SB.append_char(';'); break; case '<': - static_SB.append_char('&'); - static_SB.append_char('l'); - static_SB.append_char('t'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.append_char('&'); + kphp_runtime_context.static_SB.append_char('l'); + kphp_runtime_context.static_SB.append_char('t'); + kphp_runtime_context.static_SB.append_char(';'); break; case '>': - static_SB.append_char('&'); - static_SB.append_char('g'); - static_SB.append_char('t'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.append_char('&'); + kphp_runtime_context.static_SB.append_char('g'); + kphp_runtime_context.static_SB.append_char('t'); + kphp_runtime_context.static_SB.append_char(';'); break; default: if (str[i] < 0) { const char *utf8_str = cp1251_to_utf8_str[128 + str[i]]; - static_SB.append_unsafe(utf8_str, static_cast(strlen(utf8_str))); + kphp_runtime_context.static_SB.append_unsafe(utf8_str, static_cast(strlen(utf8_str))); } else { - static_SB.append_char(str[i]); + kphp_runtime_context.static_SB.append_char(str[i]); } } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } string f$html_entity_decode(const string &str, int64_t flags, const string &encoding) { @@ -465,59 +466,59 @@ string f$htmlspecialchars(const string &str, int64_t flags) { } const string::size_type len = str.size(); - static_SB.clean().reserve(6 * len); + kphp_runtime_context.static_SB.clean().reserve(6 * len); for (string::size_type i = 0; i < len; i++) { switch (str[i]) { case '&': - static_SB.append_char('&'); - static_SB.append_char('a'); - static_SB.append_char('m'); - static_SB.append_char('p'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.append_char('&'); + kphp_runtime_context.static_SB.append_char('a'); + kphp_runtime_context.static_SB.append_char('m'); + kphp_runtime_context.static_SB.append_char('p'); + kphp_runtime_context.static_SB.append_char(';'); break; case '"': if (!(flags & ENT_NOQUOTES)) { - static_SB.append_char('&'); - static_SB.append_char('q'); - static_SB.append_char('u'); - static_SB.append_char('o'); - static_SB.append_char('t'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.append_char('&'); + kphp_runtime_context.static_SB.append_char('q'); + kphp_runtime_context.static_SB.append_char('u'); + kphp_runtime_context.static_SB.append_char('o'); + kphp_runtime_context.static_SB.append_char('t'); + kphp_runtime_context.static_SB.append_char(';'); } else { - static_SB.append_char('"'); + kphp_runtime_context.static_SB.append_char('"'); } break; case '\'': if (flags & ENT_QUOTES) { - static_SB.append_char('&'); - static_SB.append_char('#'); - static_SB.append_char('0'); - static_SB.append_char('3'); - static_SB.append_char('9'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.append_char('&'); + kphp_runtime_context.static_SB.append_char('#'); + kphp_runtime_context.static_SB.append_char('0'); + kphp_runtime_context.static_SB.append_char('3'); + kphp_runtime_context.static_SB.append_char('9'); + kphp_runtime_context.static_SB.append_char(';'); } else { - static_SB.append_char('\''); + kphp_runtime_context.static_SB.append_char('\''); } break; case '<': - static_SB.append_char('&'); - static_SB.append_char('l'); - static_SB.append_char('t'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.append_char('&'); + kphp_runtime_context.static_SB.append_char('l'); + kphp_runtime_context.static_SB.append_char('t'); + kphp_runtime_context.static_SB.append_char(';'); break; case '>': - static_SB.append_char('&'); - static_SB.append_char('g'); - static_SB.append_char('t'); - static_SB.append_char(';'); + kphp_runtime_context.static_SB.append_char('&'); + kphp_runtime_context.static_SB.append_char('g'); + kphp_runtime_context.static_SB.append_char('t'); + kphp_runtime_context.static_SB.append_char(';'); break; default: - static_SB.append_char(str[i]); + kphp_runtime_context.static_SB.append_char(str[i]); } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } string f$htmlspecialchars_decode(const string &str, int64_t flags) { @@ -630,7 +631,7 @@ string f$ltrim(const string &s, const string &what) { string f$mysql_escape_string(const string &str) { int len = str.size(); - static_SB.clean().reserve(2 * len); + kphp_runtime_context.static_SB.clean().reserve(2 * len); for (int i = 0; i < len; i++) { switch (str[i]) { case '\0': @@ -640,13 +641,13 @@ string f$mysql_escape_string(const string &str) { case '\'': case '\"': case '\\': - static_SB.append_char('\\'); + kphp_runtime_context.static_SB.append_char('\\'); /* fallthrough */ default: - static_SB.append_char(str[i]); + kphp_runtime_context.static_SB.append_char(str[i]); } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } string f$nl2br(const string &str, bool is_xhtml) { @@ -654,20 +655,19 @@ string f$nl2br(const string &str, bool is_xhtml) { int br_len = (int)strlen(br); int len = str.size(); - - static_SB.clean().reserve((br_len + 1) * len); + kphp_runtime_context.static_SB.clean().reserve((br_len + 1) * len); for (int i = 0; i < len;) { if (str[i] == '\n' || str[i] == '\r') { - static_SB.append_unsafe(br, br_len); + kphp_runtime_context.static_SB.append_unsafe(br, br_len); if (str[i] + str[i + 1] == '\n' + '\r') { - static_SB.append_char(str[i++]); + kphp_runtime_context.static_SB.append_char(str[i++]); } } - static_SB.append_char(str[i++]); + kphp_runtime_context.static_SB.append_char(str[i++]); } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } string f$number_format(double number, int64_t decimals, const string &dec_point, const string &thousands_sep) { @@ -760,7 +760,7 @@ static double float64_from_bits(uint64_t bits) { } string f$pack(const string &pattern, const array &a) { - static_SB.clean(); + kphp_runtime_context.static_SB.clean(); int cur_arg = 0; for (int i = 0; i < (int)pattern.size();) { if (pattern[i] == '*') { @@ -817,9 +817,9 @@ string f$pack(const string &pattern, const array &a) { cnt = len; i++; } - static_SB.append(arg_str.c_str(), static_cast(min(cnt, len))); + kphp_runtime_context.static_SB.append(arg_str.c_str(), static_cast(min(cnt, len))); while (cnt > len) { - static_SB << filler; + kphp_runtime_context.static_SB << filler; cnt--; } break; @@ -841,9 +841,9 @@ string f$pack(const string &pattern, const array &a) { return {}; } if (format == 'H') { - static_SB << (char)((num_high << 4) + num_low); + kphp_runtime_context.static_SB << (char)((num_high << 4) + num_low); } else { - static_SB << (char)((num_low << 4) + num_high); + kphp_runtime_context.static_SB << (char)((num_low << 4) + num_high); } } if (cnt > 0) { @@ -857,18 +857,18 @@ string f$pack(const string &pattern, const array &a) { switch (format) { case 'c': case 'C': - static_SB << (char)(arg.to_int()); + kphp_runtime_context.static_SB << (char)(arg.to_int()); break; case 's': case 'S': case 'v': { unsigned short value = (short)arg.to_int(); - static_SB.append((const char *)&value, 2); + kphp_runtime_context.static_SB.append((const char *)&value, 2); break; } case 'n': { unsigned short value = (short)arg.to_int(); - static_SB + kphp_runtime_context.static_SB << (char)(value >> 8) << (char)(value & 255); break; @@ -879,12 +879,12 @@ string f$pack(const string &pattern, const array &a) { case 'L': case 'V': { auto value = static_cast(arg.to_int()); - static_SB.append((const char *)&value, 4); + kphp_runtime_context.static_SB.append((const char *)&value, 4); break; } case 'N': { auto value = static_cast(arg.to_int()); - static_SB + kphp_runtime_context.static_SB << (char)(value >> 24) << (char)((value >> 16) & 255) << (char)((value >> 8) & 255) @@ -893,7 +893,7 @@ string f$pack(const string &pattern, const array &a) { } case 'f': { float value = (float)arg.to_float(); - static_SB.append((const char *)&value, sizeof(float)); + kphp_runtime_context.static_SB.append((const char *)&value, sizeof(float)); break; } case 'e': @@ -906,7 +906,7 @@ string f$pack(const string &pattern, const array &a) { } else if (format == 'E') { value_byteordered = htobe64(value_byteordered); } - static_SB.append((const char *)&value_byteordered, sizeof(uint64_t)); + kphp_runtime_context.static_SB.append((const char *)&value_byteordered, sizeof(uint64_t)); break; } case 'J': @@ -922,12 +922,12 @@ string f$pack(const string &pattern, const array &a) { value_byteordered = htobe64(value_byteordered); } - static_SB.append((const char *)&value_byteordered, sizeof(unsigned long long)); + kphp_runtime_context.static_SB.append((const char *)&value_byteordered, sizeof(unsigned long long)); break; } case 'q': { int64_t value = arg.to_string().to_int(); - static_SB.append((const char *)&value, sizeof(long long)); + kphp_runtime_context.static_SB.append((const char *)&value, sizeof(long long)); break; } default: @@ -958,7 +958,7 @@ string f$pack(const string &pattern, const array &a) { php_warning("Too much arguments to call pack with format \"%s\"", pattern.c_str()); } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } string f$prepare_search_query(const string &query) { @@ -1114,7 +1114,7 @@ string f$sprintf(const string &format, const array &a) { case 'd': { int64_t arg_int = arg.to_int(); if (sign == '+' && arg_int >= 0) { - piece = (static_SB.clean() << "+" << arg_int).str(); + piece = (kphp_runtime_context.static_SB.clean() << "+" << arg_int).str(); } else { piece = string(arg_int); } @@ -1138,16 +1138,16 @@ string f$sprintf(const string &format, const array &a) { case 'G': { double arg_float = arg.to_float(); - static_SB.clean() << '%'; + kphp_runtime_context.static_SB.clean() << '%'; if (sign) { - static_SB << sign; + kphp_runtime_context.static_SB << sign; } if (precision >= 0) { - static_SB << '.' << precision; + kphp_runtime_context.static_SB << '.' << precision; } - static_SB << format[i]; + kphp_runtime_context.static_SB << format[i]; - int len = snprintf(php_buf, PHP_BUF_LEN, static_SB.c_str(), arg_float); + int len = snprintf(php_buf, PHP_BUF_LEN, kphp_runtime_context.static_SB.c_str(), arg_float); if (len >= PHP_BUF_LEN) { error_too_big = true; break; @@ -1169,13 +1169,13 @@ string f$sprintf(const string &format, const array &a) { case 's': { string arg_string = arg.to_string(); - static_SB.clean() << '%'; + kphp_runtime_context.static_SB.clean() << '%'; if (precision >= 0) { - static_SB << '.' << precision; + kphp_runtime_context.static_SB << '.' << precision; } - static_SB << 's'; + kphp_runtime_context.static_SB << 's'; - int len = snprintf(php_buf, PHP_BUF_LEN, static_SB.c_str(), arg_string.c_str()); + int len = snprintf(php_buf, PHP_BUF_LEN, kphp_runtime_context.static_SB.c_str(), arg_string.c_str()); if (len >= PHP_BUF_LEN) { error_too_big = true; break; @@ -1415,9 +1415,8 @@ string f$strip_tags(const string &str, const string &allow) { int state = 0; const string allow_low = f$strtolower(allow); - - static_SB.clean(); - static_SB_spare.clean(); + kphp_runtime_context.static_SB.clean(); + kphp_runtime_context.static_SB_spare.clean(); char lc = 0; int len = str.size(); for (int i = 0; i < len; i++) { @@ -1429,14 +1428,14 @@ string f$strip_tags(const string &str, const string &allow) { if (!in_q) { if (isspace(str[i + 1])) { if (state == 0) { - static_SB << c; + kphp_runtime_context.static_SB << c; } else if (state == 1) { - static_SB_spare << c; + kphp_runtime_context.static_SB_spare << c; } } else if (state == 0) { lc = '<'; state = 1; - static_SB_spare << '<'; + kphp_runtime_context.static_SB_spare << '<'; } else if (state == 1) { depth++; } @@ -1449,9 +1448,9 @@ string f$strip_tags(const string &str, const string &allow) { br++; } } else if (state == 1) { - static_SB_spare << c; + kphp_runtime_context.static_SB_spare << c; } else if (state == 0) { - static_SB << c; + kphp_runtime_context.static_SB << c; } break; case ')': @@ -1461,9 +1460,9 @@ string f$strip_tags(const string &str, const string &allow) { br--; } } else if (state == 1) { - static_SB_spare << c; + kphp_runtime_context.static_SB_spare << c; } else if (state == 0) { - static_SB << c; + kphp_runtime_context.static_SB << c; } break; case '>': @@ -1480,30 +1479,30 @@ string f$strip_tags(const string &str, const string &allow) { case 1: /* HTML/XML */ lc = '>'; in_q = state = 0; - static_SB_spare << '>'; - if (php_tag_find(static_SB_spare.str(), allow_low)) { - static_SB << static_SB_spare.c_str(); + kphp_runtime_context.static_SB_spare << '>'; + if (php_tag_find(kphp_runtime_context.static_SB_spare.str(), allow_low)) { + kphp_runtime_context.static_SB << kphp_runtime_context.static_SB_spare.c_str(); } - static_SB_spare.clean(); + kphp_runtime_context.static_SB_spare.clean(); break; case 2: /* PHP */ if (!br && lc != '\"' && str[i - 1] == '?') { in_q = state = 0; - static_SB_spare.clean(); + kphp_runtime_context.static_SB_spare.clean(); } break; case 3: in_q = state = 0; - static_SB_spare.clean(); + kphp_runtime_context.static_SB_spare.clean(); break; case 4: /* JavaScript/CSS/etc... */ if (i >= 2 && str[i - 1] == '-' && str[i - 2] == '-') { in_q = state = 0; - static_SB_spare.clean(); + kphp_runtime_context.static_SB_spare.clean(); } break; default: - static_SB << c; + kphp_runtime_context.static_SB << c; break; } break; @@ -1520,9 +1519,9 @@ string f$strip_tags(const string &str, const string &allow) { lc = c; } } else if (state == 0) { - static_SB << c; + kphp_runtime_context.static_SB << c; } else if (state == 1) { - static_SB_spare << c; + kphp_runtime_context.static_SB_spare << c; } if (state && i > 0 && (state == 1 || str[i - 1] != '\\') && (!in_q || c == in_q)) { if (in_q) { @@ -1539,9 +1538,9 @@ string f$strip_tags(const string &str, const string &allow) { lc = c; } else { if (state == 0) { - static_SB << c; + kphp_runtime_context.static_SB << c; } else if (state == 1) { - static_SB_spare << c; + kphp_runtime_context.static_SB_spare << c; } } break; @@ -1550,9 +1549,9 @@ string f$strip_tags(const string &str, const string &allow) { state = 4; } else { if (state == 0) { - static_SB << c; + kphp_runtime_context.static_SB << c; } else if (state == 1) { - static_SB_spare << c; + kphp_runtime_context.static_SB_spare << c; } } break; @@ -1591,15 +1590,15 @@ string f$strip_tags(const string &str, const string &allow) { /* fall-through */ default: if (state == 0) { - static_SB << c; + kphp_runtime_context.static_SB << c; } else if (state == 1) { - static_SB_spare << c; + kphp_runtime_context.static_SB_spare << c; } break; } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } template diff --git a/runtime/string_functions.h b/runtime/string_functions.h index a7c159f4ca..a0f13df801 100644 --- a/runtime/string_functions.h +++ b/runtime/string_functions.h @@ -4,8 +4,8 @@ #pragma once +#include "runtime-core/runtime-core.h" #include -#include "runtime/kphp_core.h" extern const string COLON; extern const string CP1251; diff --git a/runtime/tcp.cpp b/runtime/tcp.cpp index aa31a24c79..0176df3650 100644 --- a/runtime/tcp.cpp +++ b/runtime/tcp.cpp @@ -7,6 +7,7 @@ #include #include +#include "runtime/allocator.h" #include "runtime/critical_section.h" #include "runtime/datetime/datetime_functions.h" #include "runtime/net_events.h" diff --git a/runtime/tcp.h b/runtime/tcp.h index 6579831e7e..1772ca7a7b 100644 --- a/runtime/tcp.h +++ b/runtime/tcp.h @@ -1,6 +1,6 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" void global_init_tcp_lib(); diff --git a/runtime/tl/rpc_function.h b/runtime/tl/rpc_function.h index 65abbd832d..a2df8b19e8 100644 --- a/runtime/tl/rpc_function.h +++ b/runtime/tl/rpc_function.h @@ -4,9 +4,11 @@ #pragma once -#include "runtime/refcountable_php_classes.h" +#include + #include "common/algorithms/hashes.h" #include "common/wrappers/string_view.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" struct tl_func_base; diff --git a/runtime/tl/rpc_req_error.h b/runtime/tl/rpc_req_error.h index 4a65290e2b..e41918a864 100644 --- a/runtime/tl/rpc_req_error.h +++ b/runtime/tl/rpc_req_error.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" struct RpcError { int error_code = 0; diff --git a/runtime/tl/rpc_response.h b/runtime/tl/rpc_response.h index 4b1ae316e4..c3fb435e4c 100644 --- a/runtime/tl/rpc_response.h +++ b/runtime/tl/rpc_response.h @@ -4,8 +4,8 @@ #pragma once -#include "runtime/kphp_core.h" -#include "runtime/refcountable_php_classes.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/tl/rpc_function.h" class RpcErrorFactory { diff --git a/runtime/tl/rpc_tl_query.h b/runtime/tl/rpc_tl_query.h index 47c4485dbd..39d3b5fa7f 100644 --- a/runtime/tl/rpc_tl_query.h +++ b/runtime/tl/rpc_tl_query.h @@ -3,8 +3,11 @@ // Distributed under the GPL v3 License, see LICENSE.notice.txt #pragma once -#include "runtime/kphp_core.h" -#include "runtime/refcountable_php_classes.h" + +#include + +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" class RpcRequestResult; diff --git a/runtime/tl/tl_builtins.h b/runtime/tl/tl_builtins.h index 115153a08a..d134c58213 100644 --- a/runtime/tl/tl_builtins.h +++ b/runtime/tl/tl_builtins.h @@ -8,7 +8,7 @@ #include "common/tl/constants/common.h" -#include "runtime/include.h" +#include "runtime-core/include.h" #include "runtime/interface.h" #include "runtime/rpc.h" #include "runtime/tl/rpc_function.h" diff --git a/runtime/tl/tl_func_base.h b/runtime/tl/tl_func_base.h index 2dffce59bb..04b73612a8 100644 --- a/runtime/tl/tl_func_base.h +++ b/runtime/tl/tl_func_base.h @@ -3,8 +3,9 @@ // Distributed under the GPL v3 License, see LICENSE.notice.txt #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/tl/rpc_function.h" +#include "runtime/allocator.h" struct tl_func_base : ManagedThroughDlAllocator { virtual mixed fetch() = 0; diff --git a/runtime/tl/tl_magics_decoding.h b/runtime/tl/tl_magics_decoding.h index 168a4e215c..dfdcd1073e 100644 --- a/runtime/tl/tl_magics_decoding.h +++ b/runtime/tl/tl_magics_decoding.h @@ -6,7 +6,7 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" const char *tl_magic_convert_to_name(uint32_t magic) noexcept; array tl_magic_get_all_functions() noexcept; diff --git a/runtime/to-array-processor.h b/runtime/to-array-processor.h index 9b6cf77555..19ac809395 100644 --- a/runtime/to-array-processor.h +++ b/runtime/to-array-processor.h @@ -8,7 +8,7 @@ #include "common/mixin/not_copyable.h" #include "common/smart_ptrs/singleton.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" class ShapeKeyDemangle : vk::not_copyable { public: diff --git a/runtime/to-json-processor.h b/runtime/to-json-processor.h index d576fef63a..aa3856362f 100644 --- a/runtime/to-json-processor.h +++ b/runtime/to-json-processor.h @@ -4,9 +4,9 @@ #pragma once -#include "runtime/kphp_core.h" -#include "runtime/json-writer.h" +#include "runtime-core/runtime-core.h" #include "runtime/json-processor-utils.h" +#include "runtime/json-writer.h" template class ToJsonVisitor { diff --git a/runtime/typed_rpc.h b/runtime/typed_rpc.h index 6b57fdf5a1..88d5beaa59 100644 --- a/runtime/typed_rpc.h +++ b/runtime/typed_rpc.h @@ -3,12 +3,12 @@ // Distributed under the GPL v3 License, see LICENSE.notice.txt #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/rpc_extra_info.h" #include "runtime/tl/rpc_function.h" -#include "runtime/tl/rpc_tl_query.h" #include "runtime/tl/rpc_request.h" #include "runtime/tl/rpc_response.h" +#include "runtime/tl/rpc_tl_query.h" struct C$RpcConnection; diff --git a/runtime/uber-h3.cpp b/runtime/uber-h3.cpp index d39194af52..b5a32f8284 100644 --- a/runtime/uber-h3.cpp +++ b/runtime/uber-h3.cpp @@ -6,6 +6,8 @@ #include

+#include "runtime/allocator.h" + namespace { inline std::tuple coord2deg(GeoCoord geo_coord) noexcept { diff --git a/runtime/uber-h3.h b/runtime/uber-h3.h index fd67bedf91..80145ace52 100644 --- a/runtime/uber-h3.h +++ b/runtime/uber-h3.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" int64_t f$UberH3$$geoToH3(double latitude, double longitude, int64_t resolution) noexcept; std::tuple f$UberH3$$h3ToGeo(int64_t h3_index) noexcept; diff --git a/runtime/udp.cpp b/runtime/udp.cpp index d00524b3cb..ede9928675 100644 --- a/runtime/udp.cpp +++ b/runtime/udp.cpp @@ -10,6 +10,7 @@ #include "common/resolver.h" +#include "runtime/allocator.h" #include "runtime/critical_section.h" #include "runtime/datetime/datetime_functions.h" #include "runtime/net_events.h" diff --git a/runtime/udp.h b/runtime/udp.h index 11d195d4ac..f64087b1b2 100644 --- a/runtime/udp.h +++ b/runtime/udp.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" void global_init_udp_lib(); diff --git a/runtime/url.cpp b/runtime/url.cpp index 89c869354d..2ed7e6fc5d 100644 --- a/runtime/url.cpp +++ b/runtime/url.cpp @@ -454,22 +454,22 @@ mixed f$parse_url(const string &s, int64_t component) { } string f$rawurldecode(const string &s) { - static_SB.clean().reserve(s.size()); + kphp_runtime_context.static_SB.clean().reserve(s.size()); for (int i = 0; i < (int)s.size(); i++) { if (s[i] == '%') { int num_high = hex_to_int(s[i + 1]); if (num_high < 16) { int num_low = hex_to_int(s[i + 2]); if (num_low < 16) { - static_SB.append_char(static_cast((num_high << 4) + num_low)); + kphp_runtime_context.static_SB.append_char(static_cast((num_high << 4) + num_low)); i += 2; continue; } } } - static_SB.append_char(s[i]); + kphp_runtime_context.static_SB.append_char(s[i]); } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } @@ -484,53 +484,53 @@ static const char *good_url_symbols = "00000000000000000000000000000000";//[0-9a-zA-Z-_.] string f$rawurlencode(const string &s) { - static_SB.clean().reserve(3 * s.size()); + kphp_runtime_context.static_SB.clean().reserve(3 * s.size()); for (int i = 0; i < (int)s.size(); i++) { if (good_url_symbols[(unsigned char)s[i]] == '1') { - static_SB.append_char(s[i]); + kphp_runtime_context.static_SB.append_char(s[i]); } else { - static_SB.append_char('%'); - static_SB.append_char(uhex_digits[(s[i] >> 4) & 15]); - static_SB.append_char(uhex_digits[s[i] & 15]); + kphp_runtime_context.static_SB.append_char('%'); + kphp_runtime_context.static_SB.append_char(uhex_digits[(s[i] >> 4) & 15]); + kphp_runtime_context.static_SB.append_char(uhex_digits[s[i] & 15]); } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } string f$urldecode(const string &s) { - static_SB.clean().reserve(s.size()); + kphp_runtime_context.static_SB.clean().reserve(s.size()); for (int i = 0; i < (int)s.size(); i++) { if (s[i] == '%') { int num_high = hex_to_int(s[i + 1]); if (num_high < 16) { int num_low = hex_to_int(s[i + 2]); if (num_low < 16) { - static_SB.append_char(static_cast((num_high << 4) + num_low)); + kphp_runtime_context.static_SB.append_char(static_cast((num_high << 4) + num_low)); i += 2; continue; } } } else if (s[i] == '+') { - static_SB.append_char(' '); + kphp_runtime_context.static_SB.append_char(' '); continue; } - static_SB.append_char(s[i]); + kphp_runtime_context.static_SB.append_char(s[i]); } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } string f$urlencode(const string &s) { - static_SB.clean().reserve(3 * s.size()); + kphp_runtime_context.static_SB.clean().reserve(3 * s.size()); for (int i = 0; i < (int)s.size(); i++) { if (good_url_symbols[(unsigned char)s[i]] == '1') { - static_SB.append_char(s[i]); + kphp_runtime_context.static_SB.append_char(s[i]); } else if (s[i] == ' ') { - static_SB.append_char('+'); + kphp_runtime_context.static_SB.append_char('+'); } else { - static_SB.append_char('%'); - static_SB.append_char(uhex_digits[(s[i] >> 4) & 15]); - static_SB.append_char(uhex_digits[s[i] & 15]); + kphp_runtime_context.static_SB.append_char('%'); + kphp_runtime_context.static_SB.append_char(uhex_digits[(s[i] >> 4) & 15]); + kphp_runtime_context.static_SB.append_char(uhex_digits[s[i] & 15]); } } - return static_SB.str(); + return kphp_runtime_context.static_SB.str(); } diff --git a/runtime/url.h b/runtime/url.h index 4d1ab1dff9..bd4232aaed 100644 --- a/runtime/url.h +++ b/runtime/url.h @@ -4,7 +4,9 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" + +#include "runtime/context/runtime-context.h" Optional f$base64_decode(const string &s, bool strict = false); @@ -44,7 +46,7 @@ string http_build_query_get_param_array(const string &key, const array &a, string result; bool first = true; for (typename array::const_iterator p = a.begin(); p != a.end(); ++p) { - const string &key_value_param = http_build_query_get_param((static_SB.clean() << key << '[' << p.get_key() << ']').str(), p.get_value(), arg_separator, enc_type); + const string &key_value_param = http_build_query_get_param((kphp_runtime_context.static_SB.clean() << key << '[' << p.get_key() << ']').str(), p.get_value(), arg_separator, enc_type); if (!key_value_param.empty()) { if (!first) { result.append(arg_separator); @@ -75,7 +77,7 @@ string http_build_query_get_param(const string &key, const T &a, }; string key_encoded = encode(key); string value_encoded = encode(f$strval(a)); - return (static_SB.clean() << key_encoded << '=' << value_encoded).str(); + return (kphp_runtime_context.static_SB.clean() << key_encoded << '=' << value_encoded).str(); } } diff --git a/runtime/vkext.cpp b/runtime/vkext.cpp index e9b41fb0b8..5ed5feac53 100644 --- a/runtime/vkext.cpp +++ b/runtime/vkext.cpp @@ -10,6 +10,7 @@ #include "flex/flex.h" #include "runtime/misc.h" +#include "runtime/allocator.h" static int utf8_to_win_convert_0x400[256] = {-1, 0xa8, 0x80, 0x81, 0xaa, 0xbd, 0xb2, 0xaf, 0xa3, 0x8a, 0x8c, 0x8e, 0x8d, -1, 0xa1, 0x8f, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, diff --git a/runtime/vkext.h b/runtime/vkext.h index b4c1f779a5..b41e87d71f 100644 --- a/runtime/vkext.h +++ b/runtime/vkext.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" string f$vk_utf8_to_win(const string &text, int64_t max_len = 0, bool exit_on_error = false); diff --git a/runtime/vkext_stats.h b/runtime/vkext_stats.h index 135912d01b..03c993305b 100644 --- a/runtime/vkext_stats.h +++ b/runtime/vkext_stats.h @@ -4,8 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" - +#include "runtime-core/runtime-core.h" Optional f$vk_stats_hll_merge(const array &a); Optional f$vk_stats_hll_count(const string &hll); diff --git a/runtime/zlib.cpp b/runtime/zlib.cpp index a621a9a718..3d9d48c234 100644 --- a/runtime/zlib.cpp +++ b/runtime/zlib.cpp @@ -4,6 +4,7 @@ #include "runtime/zlib.h" +#include "runtime/context/runtime-context.h" #include "runtime/critical_section.h" #include "runtime/string_functions.h" @@ -37,7 +38,7 @@ const string_buffer *zlib_encode(const char *s, int32_t s_len, int32_t level, in strm.opaque = &buf_pos; unsigned int res_len = (unsigned int)compressBound(s_len) + 30; - static_SB.clean().reserve(res_len); + kphp_runtime_context.static_SB.clean().reserve(res_len); dl::enter_critical_section();//OK int ret = deflateInit2 (&strm, level, Z_DEFLATED, encoding, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY); @@ -45,7 +46,7 @@ const string_buffer *zlib_encode(const char *s, int32_t s_len, int32_t level, in strm.avail_in = (unsigned int)s_len; strm.next_in = reinterpret_cast (const_cast (s)); strm.avail_out = res_len; - strm.next_out = reinterpret_cast (static_SB.buffer()); + strm.next_out = reinterpret_cast (kphp_runtime_context.static_SB.buffer()); ret = deflate(&strm, Z_FINISH); deflateEnd(&strm); @@ -53,16 +54,16 @@ const string_buffer *zlib_encode(const char *s, int32_t s_len, int32_t level, in if (ret == Z_STREAM_END) { dl::leave_critical_section(); - static_SB.set_pos(static_cast(strm.total_out)); - return &static_SB; + kphp_runtime_context.static_SB.set_pos(static_cast(strm.total_out)); + return &kphp_runtime_context.static_SB; } } dl::leave_critical_section(); php_warning("Error during pack of string with length %d", s_len); - static_SB.clean(); - return &static_SB; + kphp_runtime_context.static_SB.clean(); + return &kphp_runtime_context.static_SB; } class_instance f$deflate_init(int64_t encoding, const array &options) { diff --git a/runtime/zlib.h b/runtime/zlib.h index ac6a05f7be..7da2742314 100644 --- a/runtime/zlib.h +++ b/runtime/zlib.h @@ -8,8 +8,8 @@ #include "common/wrappers/string_view.h" -#include "runtime/kphp_core.h" -#include "runtime/refcountable_php_classes.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" #include "runtime/dummy-visitor-methods.h" constexpr int64_t ZLIB_ENCODING_RAW = -0x0f; diff --git a/runtime/zstd.cpp b/runtime/zstd.cpp index b3a52a5b2f..683265411d 100644 --- a/runtime/zstd.cpp +++ b/runtime/zstd.cpp @@ -9,7 +9,7 @@ #include "common/smart_ptrs/unique_ptr_with_delete_function.h" #include "runtime/string_functions.h" - +#include "runtime/allocator.h" #include "runtime/zstd.h" namespace { diff --git a/runtime/zstd.h b/runtime/zstd.h index 3e1ada4935..752eab7410 100644 --- a/runtime/zstd.h +++ b/runtime/zstd.h @@ -4,8 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" -#include "runtime/optional.h" +#include "runtime-core/runtime-core.h" constexpr int DEFAULT_COMPRESS_LEVEL = 3; diff --git a/server/confdata-binlog-events.h b/server/confdata-binlog-events.h index bfbe8c2dc2..8aef180608 100644 --- a/server/confdata-binlog-events.h +++ b/server/confdata-binlog-events.h @@ -7,7 +7,7 @@ #include "common/type_traits/list_of_types.h" #include "server/pmemcached-binlog-interface.h" -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/memcache.h" namespace impl_ { diff --git a/server/confdata-binlog-replay.cpp b/server/confdata-binlog-replay.cpp index 7f4c6b58ce..8ea8e8f396 100644 --- a/server/confdata-binlog-replay.cpp +++ b/server/confdata-binlog-replay.cpp @@ -26,6 +26,7 @@ #include "common/tl/methods/string.h" #include "common/wrappers/string_view.h" #include "runtime/confdata-global-manager.h" +#include "runtime-core/runtime-core.h" #include "server/confdata-binlog-events.h" #include "server/confdata-stats.h" #include "server/server-log.h" diff --git a/server/confdata-stats.cpp b/server/confdata-stats.cpp index c1c4c7d360..8f9b4f335b 100644 --- a/server/confdata-stats.cpp +++ b/server/confdata-stats.cpp @@ -5,6 +5,7 @@ #include "server/confdata-stats.h" #include "common/algorithms/contains.h" +#include "runtime/memory_resource_impl/memory_resource_stats.h" namespace { @@ -84,7 +85,7 @@ const memory_resource::MemoryStats &ConfdataStats::get_memory_stats() const noex void ConfdataStats::write_stats_to(stats_t *stats) const noexcept { const auto &memory_stats = get_memory_stats(); - memory_stats.write_stats_to(stats, "confdata"); + write_memory_stats_to(memory_stats, stats, "confdata"); stats->add_gauge_stat("confdata.initial_loading_duration", to_seconds(initial_loading_time)); stats->add_gauge_stat("confdata.total_updating_time", to_seconds(total_updating_time)); diff --git a/server/database-drivers/adaptor.h b/server/database-drivers/adaptor.h index fd85754481..3dafb82e79 100644 --- a/server/database-drivers/adaptor.h +++ b/server/database-drivers/adaptor.h @@ -7,12 +7,12 @@ #include #include -#include "common/smart_ptrs/singleton.h" #include "common/mixin/not_copyable.h" +#include "common/smart_ptrs/singleton.h" +#include "runtime-core/runtime-core.h" #include "net/net-events.h" #include "runtime/critical_section.h" #include "runtime/signal_safe_hashtable.h" -#include "runtime/kphp_core.h" #include "server/database-drivers/connector.h" #include "server/database-drivers/request.h" diff --git a/server/database-drivers/mysql/mysql-connector.h b/server/database-drivers/mysql/mysql-connector.h index ec0366be24..47f82a9bf8 100644 --- a/server/database-drivers/mysql/mysql-connector.h +++ b/server/database-drivers/mysql/mysql-connector.h @@ -7,7 +7,7 @@ #include #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "server/database-drivers/connector.h" namespace database_drivers { diff --git a/server/database-drivers/mysql/mysql-request.h b/server/database-drivers/mysql/mysql-request.h index f27cde1ba7..deb12a240b 100644 --- a/server/database-drivers/mysql/mysql-request.h +++ b/server/database-drivers/mysql/mysql-request.h @@ -4,7 +4,7 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "server/database-drivers/request.h" namespace database_drivers { diff --git a/server/database-drivers/mysql/mysql.h b/server/database-drivers/mysql/mysql.h index c54907ba73..318603d22e 100644 --- a/server/database-drivers/mysql/mysql.h +++ b/server/database-drivers/mysql/mysql.h @@ -5,9 +5,9 @@ #pragma once #include "common/kprintf.h" +#include "runtime-core/runtime-core.h" #include "runtime/allocator.h" #include "runtime/critical_section.h" -#include "runtime/kphp_core.h" DECLARE_VERBOSITY(mysql); diff --git a/server/database-drivers/pgsql/pgsql-connector.h b/server/database-drivers/pgsql/pgsql-connector.h index 1adf1910a8..9d8f288a08 100644 --- a/server/database-drivers/pgsql/pgsql-connector.h +++ b/server/database-drivers/pgsql/pgsql-connector.h @@ -3,12 +3,11 @@ #include #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "server/database-drivers/connector.h" #include "server/database-drivers/pgsql/pgsql.h" #include "server/database-drivers/pgsql/pgsql-storage.h" - namespace database_drivers { class Request; diff --git a/server/database-drivers/pgsql/pgsql-request.h b/server/database-drivers/pgsql/pgsql-request.h index f7dcc3d6e2..547e1384a2 100644 --- a/server/database-drivers/pgsql/pgsql-request.h +++ b/server/database-drivers/pgsql/pgsql-request.h @@ -1,6 +1,6 @@ #pragma once -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "server/database-drivers/request.h" namespace database_drivers { diff --git a/server/database-drivers/pgsql/pgsql.h b/server/database-drivers/pgsql/pgsql.h index 7e158ff46c..fae78aecfa 100644 --- a/server/database-drivers/pgsql/pgsql.h +++ b/server/database-drivers/pgsql/pgsql.h @@ -1,9 +1,9 @@ #pragma once #include "common/kprintf.h" +#include "runtime-core/runtime-core.h" #include "runtime/allocator.h" #include "runtime/critical_section.h" -#include "runtime/kphp_core.h" DECLARE_VERBOSITY(pgsql); diff --git a/server/job-workers/job-message.h b/server/job-workers/job-message.h index fa6af2c11b..8c60491579 100644 --- a/server/job-workers/job-message.h +++ b/server/job-workers/job-message.h @@ -6,9 +6,9 @@ #include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" #include "runtime/job-workers/job-interface.h" -#include "runtime/memory_resource/unsynchronized_pool_resource.h" -#include "runtime/kphp_core.h" namespace job_workers { diff --git a/server/job-workers/job-stats.cpp b/server/job-workers/job-stats.cpp index 45885faa7a..cc777bf38e 100644 --- a/server/job-workers/job-stats.cpp +++ b/server/job-workers/job-stats.cpp @@ -5,7 +5,7 @@ #include #include -#include "runtime/memory_resource/extra-memory-pool.h" +#include "runtime-core/memory-resource/extra-memory-pool.h" #include "server/job-workers/job-stats.h" diff --git a/server/job-workers/shared-memory-manager.h b/server/job-workers/shared-memory-manager.h index 03603be804..3d80564baf 100644 --- a/server/job-workers/shared-memory-manager.h +++ b/server/job-workers/shared-memory-manager.h @@ -8,8 +8,8 @@ #include "common/mixin/not_copyable.h" #include "common/smart_ptrs/singleton.h" +#include "runtime-core/memory-resource/extra-memory-pool.h" #include "runtime/critical_section.h" -#include "runtime/memory_resource/extra-memory-pool.h" #include "server/job-workers/job-stats.h" #include "server/job-workers/job-workers-context.h" #include "server/php-engine-vars.h" diff --git a/server/php-master.cpp b/server/php-master.cpp index a7434a409c..80d5d851c0 100644 --- a/server/php-master.cpp +++ b/server/php-master.cpp @@ -50,6 +50,7 @@ #include "net/net-tcp-rpc-client.h" #include "net/net-tcp-rpc-server.h" +#include "runtime/memory_resource_impl/memory_resource_stats.h" #include "runtime/confdata-global-manager.h" #include "runtime/instance-cache.h" #include "runtime/thread-pool.h" @@ -1148,7 +1149,7 @@ STATS_PROVIDER_TAGGED(kphp_stats, 100, stats_tag_kphp_server) { stats->add_gauge_stat("server.total_json_logs_count", std::get<0>(total_workers_json_count) + master_json_logs_count); stats->add_gauge_stat("server.total_json_traces_count", std::get<1>(total_workers_json_count)); - instance_cache_get_memory_stats().write_stats_to(stats, "instance_cache"); + write_memory_stats_to(instance_cache_get_memory_stats(), stats, "instance_cache"); stats->add_gauge_stat("instance_cache.memory.buffer_swaps_ok", instance_cache_memory_swaps_ok); stats->add_gauge_stat("instance_cache.memory.buffer_swaps_fail", instance_cache_memory_swaps_fail); diff --git a/server/php-runner.cpp b/server/php-runner.cpp index 24276fd132..a6d82ede40 100644 --- a/server/php-runner.cpp +++ b/server/php-runner.cpp @@ -29,6 +29,7 @@ #include "runtime/kphp_tracing.h" #include "runtime/oom_handler.h" #include "runtime/profiler.h" +#include "runtime/php_assert.h" #include "server/json-logger.h" #include "server/php-engine-vars.h" #include "server/php-queries.h" diff --git a/server/server-stats.h b/server/server-stats.h index b637f6b5b5..81a016c15e 100644 --- a/server/server-stats.h +++ b/server/server-stats.h @@ -12,7 +12,7 @@ #include "common/smart_ptrs/singleton.h" #include "common/stats/provider.h" -#include "runtime/memory_resource/memory_resource.h" +#include "runtime-core/memory-resource/memory_resource.h" #include "server/php-runner.h" #include "server/workers-control.h" diff --git a/server/signal-handlers.cpp b/server/signal-handlers.cpp index 701cd567ca..79a07fdd80 100644 --- a/server/signal-handlers.cpp +++ b/server/signal-handlers.cpp @@ -12,6 +12,7 @@ #include "common/server/signals.h" #include "runtime/critical_section.h" #include "runtime/interface.h" +#include "runtime/php_assert.h" #include "server/json-logger.h" #include "server/php-engine-vars.h" #include "server/server-log.h" diff --git a/server/statshouse/statshouse-manager.h b/server/statshouse/statshouse-manager.h index f848f631e3..141de2e885 100644 --- a/server/statshouse/statshouse-manager.h +++ b/server/statshouse/statshouse-manager.h @@ -9,7 +9,7 @@ #include "common/dl-utils-lite.h" #include "common/mixin/not_copyable.h" -#include "runtime/memory_resource/memory_resource.h" +#include "runtime-core/memory-resource/memory_resource.h" #include "server/job-workers/job-stats.h" #include "server/statshouse/statshouse-client.h" #include "server/workers-control.h" diff --git a/tests/cpp/runtime/_runtime-tests-env.cpp b/tests/cpp/runtime/_runtime-tests-env.cpp index 2bfbc6d37c..ee911ea7b3 100644 --- a/tests/cpp/runtime/_runtime-tests-env.cpp +++ b/tests/cpp/runtime/_runtime-tests-env.cpp @@ -7,6 +7,7 @@ #include "runtime/interface.h" #include "runtime/job-workers/job-interface.h" #include "runtime/pdo/pdo_statement.h" +#include "runtime/php_assert.h" #include "runtime/tl/rpc_response.h" #include "server/php-engine-vars.h" #include "server/workers-control.h" diff --git a/tests/cpp/runtime/array-int-string-keys-collision-test.cpp b/tests/cpp/runtime/array-int-string-keys-collision-test.cpp index 09004e5baa..f0085a9f71 100644 --- a/tests/cpp/runtime/array-int-string-keys-collision-test.cpp +++ b/tests/cpp/runtime/array-int-string-keys-collision-test.cpp @@ -1,7 +1,7 @@ #include +#include "runtime-core/runtime-core.h" #include "runtime/array_functions.h" -#include "runtime/kphp_core.h" class ArrayIntStringKeysCollision : public testing::Test { protected: diff --git a/tests/cpp/runtime/array-test.cpp b/tests/cpp/runtime/array-test.cpp index c7b34c5cf3..f2ac66c4e9 100644 --- a/tests/cpp/runtime/array-test.cpp +++ b/tests/cpp/runtime/array-test.cpp @@ -1,6 +1,6 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" TEST(array_test, find_no_mutate_in_empy_array) { array arr; diff --git a/tests/cpp/runtime/kphp-type-traits-test.cpp b/tests/cpp/runtime/kphp-type-traits-test.cpp index ccdb62d3d5..ccec5ea19e 100644 --- a/tests/cpp/runtime/kphp-type-traits-test.cpp +++ b/tests/cpp/runtime/kphp-type-traits-test.cpp @@ -1,9 +1,7 @@ #include - -#include "runtime/kphp_core.h" -#include "runtime/kphp_type_traits.h" -#include "runtime/refcountable_php_classes.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" struct Stub : refcountable_php_classes { }; diff --git a/tests/cpp/runtime/memory_resource/details/memory_chunk_list-test.cpp b/tests/cpp/runtime/memory_resource/details/memory_chunk_list-test.cpp index f9ed196a2a..0921607e8f 100644 --- a/tests/cpp/runtime/memory_resource/details/memory_chunk_list-test.cpp +++ b/tests/cpp/runtime/memory_resource/details/memory_chunk_list-test.cpp @@ -1,6 +1,6 @@ #include -#include "runtime/memory_resource/details/memory_chunk_list.h" +#include "runtime-core/memory-resource/details/memory_chunk_list.h" TEST(unsynchronized_memory_chunk_list_test, empty) { memory_resource::details::memory_chunk_list mem_chunk_list; diff --git a/tests/cpp/runtime/memory_resource/details/memory_chunk_tree-test.cpp b/tests/cpp/runtime/memory_resource/details/memory_chunk_tree-test.cpp index e898abbae1..d283440fd3 100644 --- a/tests/cpp/runtime/memory_resource/details/memory_chunk_tree-test.cpp +++ b/tests/cpp/runtime/memory_resource/details/memory_chunk_tree-test.cpp @@ -2,8 +2,8 @@ #include #include -#include "runtime/memory_resource/details/memory_chunk_tree.h" -#include "runtime/memory_resource/details/memory_ordered_chunk_list.h" +#include "runtime-core/memory-resource/details/memory_chunk_tree.h" +#include "runtime-core/memory-resource/details/memory_ordered_chunk_list.h" #include "tests/cpp/runtime/memory_resource/details/test-helpers.h" TEST(memory_chunk_tree_test, empty) { diff --git a/tests/cpp/runtime/memory_resource/details/memory_ordered_chunk_list-test.cpp b/tests/cpp/runtime/memory_resource/details/memory_ordered_chunk_list-test.cpp index d98af5253f..9dc1d58687 100644 --- a/tests/cpp/runtime/memory_resource/details/memory_ordered_chunk_list-test.cpp +++ b/tests/cpp/runtime/memory_resource/details/memory_ordered_chunk_list-test.cpp @@ -3,8 +3,8 @@ #include -#include "runtime/memory_resource/details/memory_ordered_chunk_list.h" -#include "runtime/memory_resource/monotonic_buffer_resource.h" +#include "runtime-core/memory-resource/details/memory_ordered_chunk_list.h" +#include "runtime-core/memory-resource/monotonic_buffer_resource.h" #include "tests/cpp/runtime/memory_resource/details/test-helpers.h" TEST(memory_ordered_chunk_list_test, empty) { diff --git a/tests/cpp/runtime/memory_resource/details/test-helpers.h b/tests/cpp/runtime/memory_resource/details/test-helpers.h index ee96824a15..cfc69ca9fe 100644 --- a/tests/cpp/runtime/memory_resource/details/test-helpers.h +++ b/tests/cpp/runtime/memory_resource/details/test-helpers.h @@ -2,8 +2,8 @@ #include "common/wrappers/to_array.h" -#include "runtime/memory_resource/memory_resource.h" -#include "runtime/memory_resource/details/memory_chunk_list.h" +#include "runtime-core/memory-resource/details/memory_chunk_list.h" +#include "runtime-core/memory-resource/memory_resource.h" template inline auto make_offsets(const std::array &sizes) { diff --git a/tests/cpp/runtime/memory_resource/extra-memory-pool-test.cpp b/tests/cpp/runtime/memory_resource/extra-memory-pool-test.cpp index 06b57b5b99..92bae2ccb8 100644 --- a/tests/cpp/runtime/memory_resource/extra-memory-pool-test.cpp +++ b/tests/cpp/runtime/memory_resource/extra-memory-pool-test.cpp @@ -5,7 +5,7 @@ #include #include -#include "runtime/memory_resource/extra-memory-pool.h" +#include "runtime-core/memory-resource/extra-memory-pool.h" namespace mr = memory_resource; diff --git a/tests/cpp/runtime/memory_resource/unsynchronized_pool_resource-test.cpp b/tests/cpp/runtime/memory_resource/unsynchronized_pool_resource-test.cpp index cb3852699c..81dc799f2e 100644 --- a/tests/cpp/runtime/memory_resource/unsynchronized_pool_resource-test.cpp +++ b/tests/cpp/runtime/memory_resource/unsynchronized_pool_resource-test.cpp @@ -1,7 +1,7 @@ #include #include -#include "runtime/memory_resource/unsynchronized_pool_resource.h" +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" TEST(unsynchronized_pool_resource_test, uninited_state) { memory_resource::unsynchronized_pool_resource resource; diff --git a/tests/cpp/runtime/msgpack-test.cpp b/tests/cpp/runtime/msgpack-test.cpp index b8fe90032d..d55072723d 100644 --- a/tests/cpp/runtime/msgpack-test.cpp +++ b/tests/cpp/runtime/msgpack-test.cpp @@ -1,4 +1,4 @@ -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/msgpack-serialization.h" #include #include diff --git a/tests/cpp/runtime/number-string-comparison.cpp b/tests/cpp/runtime/number-string-comparison.cpp index e9c3f252b3..995a0dbe68 100644 --- a/tests/cpp/runtime/number-string-comparison.cpp +++ b/tests/cpp/runtime/number-string-comparison.cpp @@ -1,6 +1,6 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" template struct TestCaseComparison { diff --git a/tests/cpp/runtime/string-test.cpp b/tests/cpp/runtime/string-test.cpp index 6353645304..90dc4ffda9 100644 --- a/tests/cpp/runtime/string-test.cpp +++ b/tests/cpp/runtime/string-test.cpp @@ -1,6 +1,6 @@ #include -#include "runtime/kphp_core.h" +#include "runtime-core/runtime-core.h" #include "runtime/string_functions.h" TEST(string_test, test_empty) { diff --git a/tests/cpp/server/job-workers/shared-memory-manager-test.cpp b/tests/cpp/server/job-workers/shared-memory-manager-test.cpp index 0696dc50a2..a79c3005b4 100644 --- a/tests/cpp/server/job-workers/shared-memory-manager-test.cpp +++ b/tests/cpp/server/job-workers/shared-memory-manager-test.cpp @@ -6,7 +6,7 @@ #include "common/macos-ports.h" -#include "runtime/memory_resource/extra-memory-pool.h" +#include "runtime-core/memory-resource/extra-memory-pool.h" #include "server/job-workers/job-message.h" #include "server/job-workers/job-stats.h" #include "server/job-workers/job-workers-context.h" From 7496ceeebf9670501c6464497342b5b97a8b897f Mon Sep 17 00:00:00 2001 From: Vadim Sadokhov <65451602+astrophysik@users.noreply.github.com> Date: Wed, 26 Jun 2024 18:25:04 +0300 Subject: [PATCH 38/89] add runtime light (#1009) add runtime-light for integration with k2 --- CMakeLists.txt | 33 +- .../{ => kphp-full}/_functions.txt | 0 builtin-functions/{ => kphp-full}/ffi.txt | 0 builtin-functions/{ => kphp-full}/kml.txt | 0 .../{ => kphp-full}/kphp_internal.txt | 0 .../{ => kphp-full}/kphp_toggles.txt | 0 .../{ => kphp-full}/kphp_tracing.txt | 0 builtin-functions/{ => kphp-full}/pdo/PDO.php | 0 .../{ => kphp-full}/pdo/PDOStatement.php | 0 builtin-functions/{ => kphp-full}/spl.txt | 0 builtin-functions/{ => kphp-full}/uberh3.txt | 0 builtin-functions/kphp-light/functions.txt | 142 +++++ cmake/init-compilation-flags.cmake | 46 +- cmake/init-compilation-options.cmake | 3 + cmake/popular-common.cmake | 33 +- compiler/kphp2cpp.cpp | 2 +- .../memory-resource/resource_allocator.h | 17 +- runtime-core/runtime-core-context.h | 5 + runtime-core/utils/kphp-assert-core.h | 5 +- runtime-light/allocator/allocator.cmake | 2 + .../allocator/runtime-light-allocator.cpp | 137 +++++ runtime-light/component/component.cpp | 66 +++ runtime-light/component/component.h | 77 +++ runtime-light/component/image.h | 9 + runtime-light/core/core.cmake | 9 + runtime-light/core/globals/php-init-scripts.h | 14 + .../core/globals/php-script-globals.cpp | 36 ++ .../core/globals/php-script-globals.h | 60 ++ .../core/kphp-core-impl/kphp-core-context.cpp | 20 + runtime-light/coroutine/awaitable.h | 78 +++ runtime-light/coroutine/task.h | 214 +++++++ runtime-light/header.h | 279 +++++++++ .../monotonic-light-buffer-resource.cpp | 12 + runtime-light/runtime-light.cmake | 60 ++ runtime-light/runtime-light.cpp | 72 +++ runtime-light/stdlib/interface.cpp | 19 + runtime-light/stdlib/interface.h | 15 + runtime-light/stdlib/misc.cpp | 91 +++ runtime-light/stdlib/misc.h | 22 + runtime-light/stdlib/output-control.cpp | 99 ++++ runtime-light/stdlib/output-control.h | 33 ++ runtime-light/stdlib/stdlib.cmake | 8 + runtime-light/stdlib/string-functions.cpp | 101 ++++ runtime-light/stdlib/string-functions.h | 74 +++ runtime-light/stdlib/superglobals.cpp | 16 + runtime-light/stdlib/superglobals.h | 12 + runtime-light/stdlib/variable-handling.cpp | 211 +++++++ runtime-light/stdlib/variable-handling.h | 32 ++ runtime-light/streams/component-stream.cpp | 47 ++ runtime-light/streams/component-stream.h | 50 ++ runtime-light/streams/interface.cpp | 148 +++++ runtime-light/streams/interface.h | 39 ++ runtime-light/streams/streams.cmake | 5 + runtime-light/streams/streams.cpp | 201 +++++++ runtime-light/streams/streams.h | 25 + runtime-light/utils/context.cpp | 17 + runtime-light/utils/context.h | 30 + runtime-light/utils/json-functions.cpp | 537 ++++++++++++++++++ runtime-light/utils/json-functions.h | 181 ++++++ runtime-light/utils/logs.h | 13 + runtime-light/utils/panic.h | 24 + runtime-light/utils/php_assert.cpp | 65 +++ runtime-light/utils/php_assert.h | 13 + runtime-light/utils/timer.cpp | 21 + runtime-light/utils/timer.h | 20 + runtime-light/utils/to-array-processor.h | 166 ++++++ runtime-light/utils/utils.cmake | 5 + runtime/context/runtime-core-allocator.cpp | 8 + runtime/php_assert.cpp | 6 +- server/confdata-binlog-replay.cpp | 4 + tests/k2-components/dev_null.php | 3 + tests/k2-components/dev_random.php | 24 + tests/k2-components/echo.php | 12 + tests/k2-components/forward.php | 10 + tests/k2-components/oom_loop.php | 7 + tests/k2-components/proptest.php | 364 ++++++++++++ tests/k2-components/tight_loop.php | 6 + tests/k2-components/yield_loop.php | 5 + tests/tests.cmake | 7 +- 79 files changed, 4189 insertions(+), 38 deletions(-) rename builtin-functions/{ => kphp-full}/_functions.txt (100%) rename builtin-functions/{ => kphp-full}/ffi.txt (100%) rename builtin-functions/{ => kphp-full}/kml.txt (100%) rename builtin-functions/{ => kphp-full}/kphp_internal.txt (100%) rename builtin-functions/{ => kphp-full}/kphp_toggles.txt (100%) rename builtin-functions/{ => kphp-full}/kphp_tracing.txt (100%) rename builtin-functions/{ => kphp-full}/pdo/PDO.php (100%) rename builtin-functions/{ => kphp-full}/pdo/PDOStatement.php (100%) rename builtin-functions/{ => kphp-full}/spl.txt (100%) rename builtin-functions/{ => kphp-full}/uberh3.txt (100%) create mode 100644 builtin-functions/kphp-light/functions.txt create mode 100644 runtime-light/allocator/allocator.cmake create mode 100644 runtime-light/allocator/runtime-light-allocator.cpp create mode 100644 runtime-light/component/component.cpp create mode 100644 runtime-light/component/component.h create mode 100644 runtime-light/component/image.h create mode 100644 runtime-light/core/core.cmake create mode 100644 runtime-light/core/globals/php-init-scripts.h create mode 100644 runtime-light/core/globals/php-script-globals.cpp create mode 100644 runtime-light/core/globals/php-script-globals.h create mode 100644 runtime-light/core/kphp-core-impl/kphp-core-context.cpp create mode 100644 runtime-light/coroutine/awaitable.h create mode 100644 runtime-light/coroutine/task.h create mode 100644 runtime-light/header.h create mode 100644 runtime-light/memory-resource-impl/monotonic-light-buffer-resource.cpp create mode 100644 runtime-light/runtime-light.cmake create mode 100644 runtime-light/runtime-light.cpp create mode 100644 runtime-light/stdlib/interface.cpp create mode 100644 runtime-light/stdlib/interface.h create mode 100644 runtime-light/stdlib/misc.cpp create mode 100644 runtime-light/stdlib/misc.h create mode 100644 runtime-light/stdlib/output-control.cpp create mode 100644 runtime-light/stdlib/output-control.h create mode 100644 runtime-light/stdlib/stdlib.cmake create mode 100644 runtime-light/stdlib/string-functions.cpp create mode 100644 runtime-light/stdlib/string-functions.h create mode 100644 runtime-light/stdlib/superglobals.cpp create mode 100644 runtime-light/stdlib/superglobals.h create mode 100644 runtime-light/stdlib/variable-handling.cpp create mode 100644 runtime-light/stdlib/variable-handling.h create mode 100644 runtime-light/streams/component-stream.cpp create mode 100644 runtime-light/streams/component-stream.h create mode 100644 runtime-light/streams/interface.cpp create mode 100644 runtime-light/streams/interface.h create mode 100644 runtime-light/streams/streams.cmake create mode 100644 runtime-light/streams/streams.cpp create mode 100644 runtime-light/streams/streams.h create mode 100644 runtime-light/utils/context.cpp create mode 100644 runtime-light/utils/context.h create mode 100644 runtime-light/utils/json-functions.cpp create mode 100644 runtime-light/utils/json-functions.h create mode 100644 runtime-light/utils/logs.h create mode 100644 runtime-light/utils/panic.h create mode 100644 runtime-light/utils/php_assert.cpp create mode 100644 runtime-light/utils/php_assert.h create mode 100644 runtime-light/utils/timer.cpp create mode 100644 runtime-light/utils/timer.h create mode 100644 runtime-light/utils/to-array-processor.h create mode 100644 runtime-light/utils/utils.cmake create mode 100644 tests/k2-components/dev_null.php create mode 100644 tests/k2-components/dev_random.php create mode 100644 tests/k2-components/echo.php create mode 100644 tests/k2-components/forward.php create mode 100644 tests/k2-components/oom_loop.php create mode 100644 tests/k2-components/proptest.php create mode 100644 tests/k2-components/tight_loop.php create mode 100644 tests/k2-components/yield_loop.php diff --git a/CMakeLists.txt b/CMakeLists.txt index 280e882a48..1c67c518da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,28 +43,45 @@ include(${COMMON_DIR}/tl/tl.cmake) include(${COMMON_DIR}/unicode/unicode.cmake) include(${BASE_DIR}/runtime-core/runtime-core.cmake) -include(${BASE_DIR}/runtime/runtime.cmake) -include(${BASE_DIR}/server/server.cmake) +if(COMPILE_RUNTIME_LIGHT) + include(${BASE_DIR}/runtime-light/runtime-light.cmake) + file(GLOB EXCLUDE_FILES "runtime/*") +else() + include(${BASE_DIR}/runtime/runtime.cmake) + include(${BASE_DIR}/server/server.cmake) + file(GLOB EXCLUDE_FILES "runtime-light/*") +endif() include(${BASE_DIR}/compiler/compiler.cmake) - include(${BASE_DIR}/tests/tests.cmake) + if(KPHP_CUSTOM_CMAKE AND EXISTS ${OBJS_DIR}/custom_php_project.cmake) include(${OBJS_DIR}/custom_php_project.cmake) endif() add_custom_target(kphp ALL DEPENDS ${OBJS_DIR}/php_lib_version.sha256) -add_dependencies(kphp kphp2cpp kphp-full-runtime) + +if (COMPILE_RUNTIME_LIGHT) + add_dependencies(kphp kphp2cpp kphp-light-runtime) +else () + add_dependencies(kphp kphp2cpp kphp-full-runtime) +endif () install(TARGETS kphp2cpp COMPONENT KPHP RUNTIME DESTINATION ${VK_INSTALL_DIR}/bin/) install_symlink(${VK_INSTALL_DIR}/bin/kphp2cpp bin/kphp KPHP) -install(TARGETS kphp-full-runtime - COMPONENT KPHP - LIBRARY DESTINATION ${INSTALL_KPHP_SOURCE}/objs - ARCHIVE DESTINATION ${INSTALL_KPHP_SOURCE}/objs) +if (COMPILE_RUNTIME_LIGHT) + install(TARGETS kphp-light-runtime + COMPONENT KPHP + LIBRARY DESTINATION ${INSTALL_KPHP_SOURCE}/objs) +else () + install(TARGETS kphp-full-runtime + COMPONENT KPHP + LIBRARY DESTINATION ${INSTALL_KPHP_SOURCE}/objs + ARCHIVE DESTINATION ${INSTALL_KPHP_SOURCE}/objs) +endif () install(DIRECTORY ${COMMON_DIR} ${BASE_DIR}/runtime diff --git a/builtin-functions/_functions.txt b/builtin-functions/kphp-full/_functions.txt similarity index 100% rename from builtin-functions/_functions.txt rename to builtin-functions/kphp-full/_functions.txt diff --git a/builtin-functions/ffi.txt b/builtin-functions/kphp-full/ffi.txt similarity index 100% rename from builtin-functions/ffi.txt rename to builtin-functions/kphp-full/ffi.txt diff --git a/builtin-functions/kml.txt b/builtin-functions/kphp-full/kml.txt similarity index 100% rename from builtin-functions/kml.txt rename to builtin-functions/kphp-full/kml.txt diff --git a/builtin-functions/kphp_internal.txt b/builtin-functions/kphp-full/kphp_internal.txt similarity index 100% rename from builtin-functions/kphp_internal.txt rename to builtin-functions/kphp-full/kphp_internal.txt diff --git a/builtin-functions/kphp_toggles.txt b/builtin-functions/kphp-full/kphp_toggles.txt similarity index 100% rename from builtin-functions/kphp_toggles.txt rename to builtin-functions/kphp-full/kphp_toggles.txt diff --git a/builtin-functions/kphp_tracing.txt b/builtin-functions/kphp-full/kphp_tracing.txt similarity index 100% rename from builtin-functions/kphp_tracing.txt rename to builtin-functions/kphp-full/kphp_tracing.txt diff --git a/builtin-functions/pdo/PDO.php b/builtin-functions/kphp-full/pdo/PDO.php similarity index 100% rename from builtin-functions/pdo/PDO.php rename to builtin-functions/kphp-full/pdo/PDO.php diff --git a/builtin-functions/pdo/PDOStatement.php b/builtin-functions/kphp-full/pdo/PDOStatement.php similarity index 100% rename from builtin-functions/pdo/PDOStatement.php rename to builtin-functions/kphp-full/pdo/PDOStatement.php diff --git a/builtin-functions/spl.txt b/builtin-functions/kphp-full/spl.txt similarity index 100% rename from builtin-functions/spl.txt rename to builtin-functions/kphp-full/spl.txt diff --git a/builtin-functions/uberh3.txt b/builtin-functions/kphp-full/uberh3.txt similarity index 100% rename from builtin-functions/uberh3.txt rename to builtin-functions/kphp-full/uberh3.txt diff --git a/builtin-functions/kphp-light/functions.txt b/builtin-functions/kphp-light/functions.txt new file mode 100644 index 0000000000..75289b9005 --- /dev/null +++ b/builtin-functions/kphp-light/functions.txt @@ -0,0 +1,142 @@ +; + +function make_clone ($x ::: any) ::: ^1; + +/** @kphp-extern-func-info interruptible */ +function testyield() ::: void; +function check_shutdown() ::: void; + +function warning($message ::: string) ::: void; +/** @kphp-no-return */ +function critical_error($message ::: string) ::: void; + +function json_encode ($v ::: mixed, $options ::: int = 0) ::: string | false; +function json_decode ($v ::: string, $assoc ::: bool = false) ::: mixed; + +function debug_print_string($str ::: string) ::: void; + +function byte_to_int($str ::: string) ::: ?int; +function int_to_byte($v ::: int) ::: ?string; + +function set_timer(int $timeout, callable():void $callback) ::: void; diff --git a/cmake/init-compilation-flags.cmake b/cmake/init-compilation-flags.cmake index a9c8cf6fe4..35ae461a12 100644 --- a/cmake/init-compilation-flags.cmake +++ b/cmake/init-compilation-flags.cmake @@ -1,20 +1,37 @@ include_guard(GLOBAL) +# Light runtime require c++20 if(CMAKE_CXX_COMPILER_ID MATCHES Clang) - check_compiler_version(clang 10.0.0) + if (COMPILE_RUNTIME_LIGHT) + check_compiler_version(clang 14.0.0) + else() + check_compiler_version(clang 10.0.0) + endif() set(COMPILER_CLANG True) elseif(CMAKE_CXX_COMPILER_ID MATCHES GNU) - check_compiler_version(gcc 8.3.0) + if (COMPILE_RUNTIME_LIGHT) + check_compiler_version(gcc 10.1.0) + else() + check_compiler_version(gcc 8.3.0) + endif() set(COMPILER_GCC True) endif() -set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to") +if(COMPILE_RUNTIME_LIGHT) + set(REQUIRED_CMAKE_CXX_STANDARD 20) +else() + set(REQUIRED_CMAKE_CXX_STANDARD 17) +endif() + +set(CMAKE_CXX_STANDARD ${REQUIRED_CMAKE_CXX_STANDARD} CACHE STRING "C++ standard to conform to") set(CMAKE_CXX_EXTENSIONS OFF) -if (CMAKE_CXX_STANDARD LESS 17) - message(FATAL_ERROR "c++17 expected at least!") +if (CMAKE_CXX_STANDARD LESS ${REQUIRED_CMAKE_CXX_STANDARD}) + message(FATAL_ERROR "c++${REQUIRED_CMAKE_CXX_STANDARD} expected at least!") endif() + cmake_print_variables(CMAKE_CXX_STANDARD) + if(APPLE) add_definitions(-D_XOPEN_SOURCE) if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") @@ -101,3 +118,22 @@ add_link_options(-rdynamic -L/usr/local/lib -ggdb) add_definitions(-D_GNU_SOURCE) # prevents the `build` directory to be appeared in symbols, it's necessary for remote debugging with path mappings add_compile_options(-fdebug-prefix-map="${CMAKE_BINARY_DIR}=${CMAKE_SOURCE_DIR}") + +# Light runtime uses C++20 coroutines heavily, so they are required +if(COMPILE_RUNTIME_LIGHT) + get_directory_property(TRY_COMPILE_COMPILE_OPTIONS COMPILE_OPTIONS) + string (REPLACE ";" " " TRY_COMPILE_COMPILE_OPTIONS "${TRY_COMPILE_COMPILE_OPTIONS}") + file(WRITE "${PROJECT_BINARY_DIR}/check_coroutine_include.cpp" + "#include\n" + "int main() {}\n") + try_compile( + HAS_COROUTINE + "${PROJECT_BINARY_DIR}/tmp" + "${PROJECT_BINARY_DIR}/check_coroutine_include.cpp" + COMPILE_DEFINITIONS "${TRY_COMPILE_COMPILE_OPTIONS}" + ) + if(NOT HAS_COROUTINE) + message(FATAL_ERROR "Compiler or libstdc++ does not support coroutines") + endif() + file(REMOVE "${PROJECT_BINARY_DIR}/check_coroutine_include.cpp") +endif() diff --git a/cmake/init-compilation-options.cmake b/cmake/init-compilation-options.cmake index 2354f69f4f..ba0aa36edf 100644 --- a/cmake/init-compilation-options.cmake +++ b/cmake/init-compilation-options.cmake @@ -33,3 +33,6 @@ cmake_print_variables(KPHP_TESTS) option(KPHP_CUSTOM_CMAKE "Use CMakeLists.txt of custom php project" OFF) cmake_print_variables(KPHP_CUSTOM_CMAKE) + +option(COMPILE_RUNTIME_LIGHT "Compile runtime-light (it require c++20)" OFF) +cmake_print_variables(COMPILE_RUNTIME_LIGHT) diff --git a/cmake/popular-common.cmake b/cmake/popular-common.cmake index 13b7836883..185a11efe9 100644 --- a/cmake/popular-common.cmake +++ b/cmake/popular-common.cmake @@ -1,34 +1,41 @@ include_guard(GLOBAL) -prepend(POPULAR_COMMON_SOURCES ${COMMON_DIR}/ +prepend(LIGHT_COMMON_SOURCES ${COMMON_DIR}/ algorithms/simd-int-to-string.cpp - server/limits.cpp - server/signals.cpp - server/relogin.cpp - server/crash-dump.cpp - server/engine-settings.cpp - stats/buffer.cpp - stats/provider.cpp +) + +prepend(POPULAR_COMMON_SOURCES ${COMMON_DIR}/ resolver.cpp - kprintf.cpp precise-time.cpp cpuid.cpp crc32.cpp crc32c.cpp options.cpp - kernel-version.cpp secure-bzero.cpp crc32_${CMAKE_SYSTEM_PROCESSOR}.cpp crc32c_${CMAKE_SYSTEM_PROCESSOR}.cpp + version-string.cpp + kernel-version.cpp + kprintf.cpp + rpc-headers.cpp parallel/counter.cpp parallel/maximum.cpp parallel/thread-id.cpp parallel/limit-counter.cpp - version-string.cpp - rpc-headers.cpp) + server/limits.cpp + server/signals.cpp + server/relogin.cpp + server/crash-dump.cpp + server/engine-settings.cpp + stats/buffer.cpp + stats/provider.cpp) if(APPLE) list(APPEND POPULAR_COMMON_SOURCES ${COMMON_DIR}/macos-ports.cpp) endif() -vk_add_library(popular_common OBJECT ${POPULAR_COMMON_SOURCES}) +vk_add_library(light_common OBJECT ${LIGHT_COMMON_SOURCES}) +set_property(TARGET light_common PROPERTY POSITION_INDEPENDENT_CODE ON) +vk_add_library(popular_common OBJECT ${POPULAR_COMMON_SOURCES} ${LIGHT_COMMON_SOURCES}) +set_property(TARGET popular_common PROPERTY POSITION_INDEPENDENT_CODE ON) + diff --git a/compiler/kphp2cpp.cpp b/compiler/kphp2cpp.cpp index 94944e16d4..33589f832e 100644 --- a/compiler/kphp2cpp.cpp +++ b/compiler/kphp2cpp.cpp @@ -212,7 +212,7 @@ int main(int argc, char *argv[]) { parser.add("Path to kphp source", settings->kphp_src_path, 's', "source-path", "KPHP_PATH", get_default_kphp_path()); parser.add("Internal file with the list of supported PHP functions", settings->functions_file, - 'f', "functions-file", "KPHP_FUNCTIONS", "${KPHP_PATH}/builtin-functions/_functions.txt"); + 'f', "functions-file", "KPHP_FUNCTIONS", "${KPHP_PATH}/builtin-functions/kphp-full/_functions.txt"); parser.add("File with kphp runtime sha256 hash", settings->runtime_sha256_file, "runtime-sha256", "KPHP_RUNTIME_SHA256", "${KPHP_PATH}/objs/php_lib_version.sha256"); parser.add("The output binary type: server, cli or lib", settings->mode, diff --git a/runtime-core/memory-resource/resource_allocator.h b/runtime-core/memory-resource/resource_allocator.h index d96cd1aee3..a4bab87535 100644 --- a/runtime-core/memory-resource/resource_allocator.h +++ b/runtime-core/memory-resource/resource_allocator.h @@ -4,6 +4,8 @@ #pragma once #include +#include +#include #include "common/wrappers/likely.h" @@ -31,8 +33,7 @@ class resource_allocator { value_type *allocate(size_t size, void const * = nullptr) { static_assert(sizeof(value_type) <= max_value_type_size(), "memory limit"); - php_assert(size == 1); - auto result = static_cast(memory_resource_.allocate(sizeof(value_type))); + auto result = static_cast(memory_resource_.allocate(sizeof(value_type) * size)); if (unlikely(!result)) { php_critical_error("not enough memory to continue"); } @@ -41,8 +42,7 @@ class resource_allocator { void deallocate(value_type *mem, size_t size) { static_assert(sizeof(value_type) <= max_value_type_size(), "memory limit"); - php_assert(size == 1); - memory_resource_.deallocate(mem, sizeof(value_type)); + memory_resource_.deallocate(mem, sizeof(value_type) * size); } static constexpr size_t max_value_type_size() { @@ -62,11 +62,20 @@ class resource_allocator { }; namespace stl { +template, class KeyEqual = std::equal_to> +using unordered_map = std::unordered_map, Resource>>; + template> using map = std::map, Resource>>; template> using multimap = std::multimap, Resource>>; + +template +using deque = std::deque>; + +template +using queue = std::queue>>; } // namespace stl } // namespace memory_resource diff --git a/runtime-core/runtime-core-context.h b/runtime-core/runtime-core-context.h index b8ef96c820..a574e49280 100644 --- a/runtime-core/runtime-core-context.h +++ b/runtime-core/runtime-core-context.h @@ -14,6 +14,9 @@ struct RuntimeAllocator { static RuntimeAllocator& current() noexcept; + RuntimeAllocator() = default; + RuntimeAllocator(size_t script_mem_size, size_t oom_handling_mem_size); + void init(void * buffer, size_t script_mem_size, size_t oom_handling_mem_size); void free(); @@ -23,6 +26,7 @@ struct RuntimeAllocator { void free_script_memory(void *mem, size_t size) noexcept; void * alloc_global_memory(size_t size) noexcept; + void * alloc0_global_memory(size_t size) noexcept; void * realloc_global_memory(void *mem, size_t new_size, size_t old_size) noexcept; void free_global_memory(void *mem, size_t size) noexcept; @@ -32,6 +36,7 @@ struct RuntimeAllocator { struct KphpCoreContext { static KphpCoreContext& current() noexcept; + void init(); void free(); diff --git a/runtime-core/utils/kphp-assert-core.h b/runtime-core/utils/kphp-assert-core.h index 44024d5508..ee83cb5d01 100644 --- a/runtime-core/utils/kphp-assert-core.h +++ b/runtime-core/utils/kphp-assert-core.h @@ -4,12 +4,13 @@ #include "common/wrappers/likely.h" +void php_debug(char const *message, ...) __attribute__ ((format (printf, 1, 2))); void php_notice(char const *message, ...) __attribute__ ((format (printf, 1, 2))); void php_warning(char const *message, ...) __attribute__ ((format (printf, 1, 2))); void php_error(char const *message, ...) __attribute__ ((format (printf, 1, 2))); [[noreturn]] void php_assert__(const char *msg, const char *file, int line); -[[noreturn]] void raise_php_assert_signal__(); +[[noreturn]] void critical_error_handler(); #define php_assert(EX) do { \ if (unlikely(!(EX))) { \ @@ -19,5 +20,5 @@ void php_error(char const *message, ...) __attribute__ ((format (printf, 1, 2))) #define php_critical_error(format, ...) do { \ php_error ("Critical error \"" format "\" in file %s on line %d", ##__VA_ARGS__, __FILE__, __LINE__); \ - raise_php_assert_signal__(); \ + critical_error_handler(); \ } while(0) diff --git a/runtime-light/allocator/allocator.cmake b/runtime-light/allocator/allocator.cmake new file mode 100644 index 0000000000..7ec6677fe7 --- /dev/null +++ b/runtime-light/allocator/allocator.cmake @@ -0,0 +1,2 @@ +set(RUNTIME_ALLOCATOR_SRC + ${BASE_DIR}/runtime-light/allocator/runtime-light-allocator.cpp) diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp new file mode 100644 index 0000000000..1e82f85392 --- /dev/null +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -0,0 +1,137 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-core/runtime-core.h" +#include "runtime-light/component/component.h" +#include "runtime-light/utils/panic.h" + +static constexpr size_t MIN_REQ_EXTRA_MEM_SIZE = 16 * 1024u; + +namespace { +bool is_script_allocator_available() { + return get_component_context() != nullptr; +} + +void request_extra_memory(size_t requested_size) { + ComponentState &rt_ctx = *get_component_context(); + size_t extra_mem_size = std::max(MIN_REQ_EXTRA_MEM_SIZE, requested_size); // extra mem size should be greater than max chunk block size + void *extra_mem = get_platform_context()->allocator.alloc(extra_mem_size); + if (extra_mem == nullptr) { + php_error("script OOM"); + } + rt_ctx.runtime_allocator.memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); +} +} // namespace + +RuntimeAllocator::RuntimeAllocator(size_t script_mem_size, size_t oom_handling_mem_size) { + void *buffer = alloc_global_memory(script_mem_size); + php_assert(buffer != nullptr); + memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); +} + +RuntimeAllocator &RuntimeAllocator::current() noexcept { + return get_component_context()->runtime_allocator; +} + +void RuntimeAllocator::init(void *buffer, size_t script_mem_size, size_t oom_handling_mem_size) { + php_assert(buffer != nullptr); + memory_resource.init(buffer, script_mem_size, oom_handling_mem_size); +} + +void RuntimeAllocator::free() { + const Allocator &pt_ctx = get_platform_context()->allocator; + + auto *extra_memory = memory_resource.get_extra_memory_head(); + while (extra_memory->get_pool_payload_size() != 0) { + auto *releasing_extra_memory = extra_memory; + extra_memory = extra_memory->next_in_chain; + pt_ctx.free(releasing_extra_memory); + } + pt_ctx.free(memory_resource.memory_begin()); +} + +void *RuntimeAllocator::alloc_script_memory(size_t size) noexcept { + php_assert(size); + if (unlikely(!is_script_allocator_available())) { + return alloc_global_memory(size); + } + + ComponentState &rt_ctx = *get_component_context(); + + void *ptr = rt_ctx.runtime_allocator.memory_resource.allocate(size); + if (ptr == nullptr) { + request_extra_memory(size * 2); + ptr = rt_ctx.runtime_allocator.memory_resource.allocate(size); + php_assert(ptr != nullptr); + } + return ptr; +} + +void *RuntimeAllocator::alloc0_script_memory(size_t size) noexcept { + php_assert(size); + if (unlikely(!is_script_allocator_available())) { + return alloc0_global_memory(size); + } + + ComponentState &rt_ctx = *get_component_context(); + void *ptr = rt_ctx.runtime_allocator.memory_resource.allocate0(size); + if (ptr == nullptr) { + request_extra_memory(size * 2); + ptr = rt_ctx.runtime_allocator.memory_resource.allocate0(size); + php_assert(ptr != nullptr); + } + return ptr; +} + +void *RuntimeAllocator::realloc_script_memory(void *mem, size_t new_size, size_t old_size) noexcept { + php_assert(new_size > old_size); + if (unlikely(!is_script_allocator_available())) { + return realloc_global_memory(mem, new_size, old_size); + } + + ComponentState &rt_ctx = *get_component_context(); + void *ptr = rt_ctx.runtime_allocator.memory_resource.reallocate(mem, new_size, old_size); + if (ptr == nullptr) { + request_extra_memory(new_size * 2); + ptr = rt_ctx.runtime_allocator.memory_resource.reallocate(mem, new_size, old_size); + php_assert(ptr != nullptr); + } + return ptr; +} + +void RuntimeAllocator::free_script_memory(void *mem, size_t size) noexcept { + php_assert(size); + + ComponentState &rt_ctx = *get_component_context(); + return rt_ctx.runtime_allocator.memory_resource.deallocate(mem, size); +} + +void *RuntimeAllocator::alloc_global_memory(size_t size) noexcept { + void *ptr = get_platform_context()->allocator.alloc(size); + if (unlikely(ptr == nullptr)) { + critical_error_handler(); + } + return ptr; +} + +void *RuntimeAllocator::alloc0_global_memory(size_t size) noexcept { + void *ptr = get_platform_context()->allocator.alloc(size); + if (unlikely(ptr == nullptr)) { + critical_error_handler(); + } + memset(ptr, 0, size); + return ptr; +} + +void *RuntimeAllocator::realloc_global_memory(void *mem, size_t new_size, size_t) noexcept { + void *ptr = get_platform_context()->allocator.realloc(mem, new_size); + if (unlikely(ptr == nullptr)) { + critical_error_handler(); + } + return ptr; +} + +void RuntimeAllocator::free_global_memory(void *mem, size_t) noexcept { + get_platform_context()->allocator.free(mem); +} \ No newline at end of file diff --git a/runtime-light/component/component.cpp b/runtime-light/component/component.cpp new file mode 100644 index 0000000000..72f282f749 --- /dev/null +++ b/runtime-light/component/component.cpp @@ -0,0 +1,66 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/component/component.h" +#include "runtime-light/core/globals/php-init-scripts.h" + +void ComponentState::resume_if_was_rescheduled() { + if (poll_status == PollStatus::PollReschedule) { + // If component was suspended by please yield and there is no awaitable streams + main_thread(); + } +} + +bool ComponentState::is_stream_already_being_processed(uint64_t stream_d) { + return opened_streams.contains(stream_d); +} + +void ComponentState::resume_if_wait_stream(uint64_t stream_d, StreamStatus status) { + if (is_stream_timer(stream_d)) { + process_timer(stream_d); + } else { + process_stream(stream_d, status); + } +} + +void ComponentState::process_new_input_stream(uint64_t stream_d) { + bool already_pending = std::find(incoming_pending_queries.begin(), incoming_pending_queries.end(), stream_d) != incoming_pending_queries.end(); + if (!already_pending) { + php_debug("got new pending query %lu", stream_d); + incoming_pending_queries.push_back(stream_d); + } + if (wait_incoming_stream) { + php_debug("start process pending query %lu", stream_d); + main_thread(); + } +} + +void ComponentState::init_script_execution() { + kphp_core_context.init(); + init_php_scripts_in_each_worker(php_script_mutable_globals_singleton, k_main); + main_thread = k_main.get_handle(); +} + +bool ComponentState::is_stream_timer(uint64_t stream_d) { + return timer_callbacks.contains(stream_d); +} + +void ComponentState::process_timer(uint64_t stream_d) { + get_platform_context()->free_descriptor(stream_d); + timer_callbacks[stream_d](); + timer_callbacks.erase(stream_d); + opened_streams.erase(stream_d); +} + +void ComponentState::process_stream(uint64_t stream_d, StreamStatus status) { + auto expected_status = opened_streams[stream_d]; + if ((expected_status == StreamRuntimeStatus::WBlocked && status.write_status != IOBlocked) + || (expected_status == StreamRuntimeStatus::RBlocked && status.read_status != IOBlocked)) { + php_debug("resume on waited query %lu", stream_d); + auto suspend_point = awaiting_coroutines[stream_d]; + awaiting_coroutines.erase(stream_d); + php_assert(awaiting_coroutines.empty()); + suspend_point(); + } +} diff --git a/runtime-light/component/component.h b/runtime-light/component/component.h new file mode 100644 index 0000000000..638274ac98 --- /dev/null +++ b/runtime-light/component/component.h @@ -0,0 +1,77 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include + +#include "runtime-core/memory-resource/resource_allocator.h" +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" +#include "runtime-core/runtime-core.h" + +#include "runtime-light/core/globals/php-script-globals.h" +#include "runtime-light/coroutine/task.h" +#include "runtime-light/stdlib/output-control.h" +#include "runtime-light/stdlib/superglobals.h" +#include "runtime-light/streams/streams.h" +#include "runtime-light/utils/context.h" + +struct ComponentState { + template + using unordered_map = memory_resource::stl::unordered_map; + template + using deque = memory_resource::stl::deque; + static constexpr int INIT_RUNTIME_ALLOCATOR_SIZE = 16 * 1024u; + + ComponentState() + : runtime_allocator(INIT_RUNTIME_ALLOCATOR_SIZE, 0) + , php_script_mutable_globals_singleton(runtime_allocator.memory_resource) + , opened_streams(unordered_map::allocator_type{runtime_allocator.memory_resource}) + , awaiting_coroutines(unordered_map>::allocator_type{runtime_allocator.memory_resource}) + , timer_callbacks(unordered_map>::allocator_type{runtime_allocator.memory_resource}) + , incoming_pending_queries(deque::allocator_type{runtime_allocator.memory_resource}) {} + + ~ComponentState() = default; + + inline bool not_finished() const noexcept { + return poll_status != PollStatus::PollFinishedOk && poll_status != PollStatus::PollFinishedError; + } + + void resume_if_was_rescheduled(); + + bool is_stream_already_being_processed(uint64_t stream_d); + + void resume_if_wait_stream(uint64_t stream_d, StreamStatus status); + + void process_new_input_stream(uint64_t stream_d); + + void init_script_execution(); + + RuntimeAllocator runtime_allocator; + task_t k_main; + Response response; + PhpScriptMutableGlobals php_script_mutable_globals_singleton; + + PollStatus poll_status = PollStatus::PollReschedule; + uint64_t standard_stream = 0; + std::coroutine_handle<> main_thread; + bool wait_incoming_stream = false; + + unordered_map opened_streams; // подумать про необходимость opened_streams. Объединить с awaiting_coroutines + unordered_map> awaiting_coroutines; + unordered_map> timer_callbacks; + deque incoming_pending_queries; + + KphpCoreContext kphp_core_context; + +private: + bool is_stream_timer(uint64_t stream_d); + + void process_timer(uint64_t stream_d); + + void process_stream(uint64_t stream_d, StreamStatus status); +}; diff --git a/runtime-light/component/image.h b/runtime-light/component/image.h new file mode 100644 index 0000000000..ec75c8d9c1 --- /dev/null +++ b/runtime-light/component/image.h @@ -0,0 +1,9 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-light/header.h" + +struct ImageState {}; diff --git a/runtime-light/core/core.cmake b/runtime-light/core/core.cmake new file mode 100644 index 0000000000..5ee7b94dac --- /dev/null +++ b/runtime-light/core/core.cmake @@ -0,0 +1,9 @@ +prepend(RUNTIME_LANGUAGE_SRC ${BASE_DIR}/runtime-light/core/globals/ + php-script-globals.cpp) + +prepend(RUNTIME_KPHP_CORE_CONTEXT_SRC ${BASE_DIR}/runtime-light/core/kphp-core-impl/ + kphp-core-context.cpp) + + +set(RUNTIME_CORE_SRC ${RUNTIME_LANGUAGE_SRC} + ${RUNTIME_KPHP_CORE_CONTEXT_SRC}) \ No newline at end of file diff --git a/runtime-light/core/globals/php-init-scripts.h b/runtime-light/core/globals/php-init-scripts.h new file mode 100644 index 0000000000..6acf8111cc --- /dev/null +++ b/runtime-light/core/globals/php-init-scripts.h @@ -0,0 +1,14 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +class PhpScriptMutableGlobals; + +/// Initializes const variables represented as globals C++ symbols. Definition is generated by compiler. +void init_php_scripts_once_in_master() noexcept; +/// Initializes mutable globals. Definition is generated by compiler. +void init_php_scripts_in_each_worker(PhpScriptMutableGlobals &php_globals, task_t &run) noexcept; diff --git a/runtime-light/core/globals/php-script-globals.cpp b/runtime-light/core/globals/php-script-globals.cpp new file mode 100644 index 0000000000..d9f2421d82 --- /dev/null +++ b/runtime-light/core/globals/php-script-globals.cpp @@ -0,0 +1,36 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "php-script-globals.h" + +#include "runtime-light/component/component.h" + +PhpScriptMutableGlobals &PhpScriptMutableGlobals::current() noexcept { + return get_component_context()->php_script_mutable_globals_singleton; +} + +void PhpScriptMutableGlobals::once_alloc_linear_mem(unsigned int n_bytes) { + php_assert(g_linear_mem == nullptr); + g_linear_mem = static_cast(RuntimeAllocator::current().alloc0_global_memory(n_bytes)); +} + +void PhpScriptMutableGlobals::once_alloc_linear_mem(const char *lib_name, unsigned int n_bytes) { + int64_t key_lib_name = string_hash(lib_name, strlen(lib_name)); + php_assert(libs_linear_mem.find(key_lib_name) == libs_linear_mem.end()); + libs_linear_mem[key_lib_name] = static_cast(RuntimeAllocator::current().alloc0_global_memory(n_bytes)); +} + +char *PhpScriptMutableGlobals::get_linear_mem(const char *lib_name) const { + int64_t key_lib_name = string_hash(lib_name, strlen(lib_name)); + auto found = libs_linear_mem.find(key_lib_name); + php_assert(found != libs_linear_mem.end()); + return found->second; +} + +char *PhpScriptMutableGlobals::mem_for_lib(const char *lib_name) const { + int64_t key_lib_name = string_hash(lib_name, strlen(lib_name)); + auto found = libs_linear_mem.find(key_lib_name); + php_assert(found != libs_linear_mem.end()); + return found->second; +} diff --git a/runtime-light/core/globals/php-script-globals.h b/runtime-light/core/globals/php-script-globals.h new file mode 100644 index 0000000000..5b843dae17 --- /dev/null +++ b/runtime-light/core/globals/php-script-globals.h @@ -0,0 +1,60 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-core/memory-resource/resource_allocator.h" +#include "runtime-core/memory-resource/unsynchronized_pool_resource.h" +#include "runtime-core/runtime-core.h" + +struct PhpScriptBuiltInSuperGlobals { + // variables below are PHP language superglobals + mixed v$_SERVER; + mixed v$_GET; + mixed v$_POST; + mixed v$_ENV; + mixed v$_FILES; + mixed v$_COOKIE; + mixed v$_REQUEST; + + // variables below are not superglobals of the PHP language, but since they are set by runtime, + // the compiler is also aware about them + mixed v$argc; + mixed v$argv; + string v$d$PHP_SAPI; // define('PHP_SAPI') +}; + +class PhpScriptMutableGlobals { + template + using unordered_map = memory_resource::stl::unordered_map; + char *g_linear_mem{nullptr}; + unordered_map libs_linear_mem; + PhpScriptBuiltInSuperGlobals superglobals; + +public: + PhpScriptMutableGlobals(memory_resource::unsynchronized_pool_resource &resource) + : libs_linear_mem(unordered_map::allocator_type{resource}) {} + + static PhpScriptMutableGlobals ¤t() noexcept; + + void once_alloc_linear_mem(unsigned int n_bytes); + void once_alloc_linear_mem(const char *lib_name, unsigned int n_bytes); + + char *get_linear_mem() const { + return g_linear_mem; + } + char *get_linear_mem(const char *lib_name) const; + + char *mem() const { + return g_linear_mem; + } + char *mem_for_lib(const char *lib_name) const; + + PhpScriptBuiltInSuperGlobals &get_superglobals() { + return superglobals; + } + const PhpScriptBuiltInSuperGlobals &get_superglobals() const { + return superglobals; + } +}; diff --git a/runtime-light/core/kphp-core-impl/kphp-core-context.cpp b/runtime-light/core/kphp-core-impl/kphp-core-context.cpp new file mode 100644 index 0000000000..01fa1afd7b --- /dev/null +++ b/runtime-light/core/kphp-core-impl/kphp-core-context.cpp @@ -0,0 +1,20 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-core/runtime-core.h" +#include "runtime-light/component/component.h" +#include "runtime-light/utils/context.h" + +constexpr string_size_type initial_minimum_string_buffer_length = 1024; +constexpr string_size_type initial_maximum_string_buffer_length = (1 << 24); + +KphpCoreContext &KphpCoreContext::current() noexcept { + return get_component_context()->kphp_core_context; +} + +void KphpCoreContext::init() { + init_string_buffer_lib(initial_minimum_string_buffer_length, initial_maximum_string_buffer_length); +} + +void KphpCoreContext::free() {} diff --git a/runtime-light/coroutine/awaitable.h b/runtime-light/coroutine/awaitable.h new file mode 100644 index 0000000000..e239cd559d --- /dev/null +++ b/runtime-light/coroutine/awaitable.h @@ -0,0 +1,78 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-light/component/component.h" +#include "runtime-light/utils/logs.h" + +struct blocked_operation_t { + uint64_t awaited_stream; + + blocked_operation_t(uint64_t stream_d) + : awaited_stream(stream_d) {} + + constexpr bool await_ready() const noexcept { + return false; + } + + void await_resume() const noexcept { + ComponentState &ctx = *get_component_context(); + ctx.opened_streams[awaited_stream] = StreamRuntimeStatus::NotBlocked; + } +}; + +struct read_blocked_t : blocked_operation_t { + void await_suspend(std::coroutine_handle<> h) const noexcept { + php_debug("blocked read on stream %lu", awaited_stream); + ComponentState &ctx = *get_component_context(); + ctx.poll_status = PollStatus::PollBlocked; + ctx.opened_streams[awaited_stream] = StreamRuntimeStatus::RBlocked; + ctx.awaiting_coroutines[awaited_stream] = h; + } +}; + +struct write_blocked_t : blocked_operation_t { + void await_suspend(std::coroutine_handle<> h) const noexcept { + php_debug("blocked write on stream %lu", awaited_stream); + ComponentState &ctx = *get_component_context(); + ctx.poll_status = PollStatus::PollBlocked; + ctx.opened_streams[awaited_stream] = StreamRuntimeStatus::WBlocked; + ctx.awaiting_coroutines[awaited_stream] = h; + } +}; + +struct test_yield_t { + bool await_ready() const noexcept { + return !get_platform_context()->please_yield.load(); + } + + void await_suspend(std::coroutine_handle<> h) const noexcept { + ComponentState &ctx = *get_component_context(); + ctx.poll_status = PollStatus::PollReschedule; + ctx.main_thread = h; + } + + constexpr void await_resume() const noexcept {} +}; + +struct wait_incoming_query_t { + bool await_ready() const noexcept { + return !get_component_context()->incoming_pending_queries.empty(); + } + + void await_suspend(std::coroutine_handle<> h) const noexcept { + ComponentState &ctx = *get_component_context(); + php_assert(ctx.standard_stream == 0); + ctx.main_thread = h; + ctx.wait_incoming_stream = true; + ctx.poll_status = PollBlocked; + } + + void await_resume() const noexcept { + get_component_context()->wait_incoming_stream = false; + } +}; diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h new file mode 100644 index 0000000000..1a6db0bec0 --- /dev/null +++ b/runtime-light/coroutine/task.h @@ -0,0 +1,214 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include + +#include "common/containers/final_action.h" +#include "runtime-light/utils/context.h" + +#if __clang_major__ > 7 +#define CPPCORO_COMPILER_SUPPORTS_SYMMETRIC_TRANSFER +#endif + +struct task_base_t { + task_base_t() = default; + + explicit task_base_t(std::coroutine_handle<> handle) + : handle_address{handle.address()} {} + + task_base_t(task_base_t &&task) noexcept + : handle_address{std::exchange(task.handle_address, nullptr)} {} + + task_base_t(const task_base_t &task) = delete; + + task_base_t &operator=(task_base_t &&task) noexcept { + std::swap(handle_address, task.handle_address); + return *this; + } + + task_base_t &operator=(const task_base_t &task) = delete; + + ~task_base_t() { + if (handle_address) { + std::coroutine_handle<>::from_address(handle_address).destroy(); + } + } + + bool done() const { + assert(handle_address); + return std::coroutine_handle<>::from_address(handle_address).done(); + } + +protected: + void *handle_address{nullptr}; +}; + +template +struct task_t : public task_base_t { + template F> + struct promise_non_void_t; + struct promise_void_t; + + using promise_type = std::conditional_t{}, promise_non_void_t, promise_void_t>; + + using task_base_t::task_base_t; + + struct promise_base_t { + task_t get_return_object() noexcept { + return task_t{std::coroutine_handle::from_promise(*static_cast(this))}; + } + + std::suspend_always initial_suspend() { + return {}; + } + + struct final_suspend_t { + final_suspend_t() = default; + + bool await_ready() const noexcept { + return false; + } + + std::coroutine_handle<> await_suspend(std::coroutine_handle h) const noexcept { +#ifdef CPPCORO_COMPILER_SUPPORTS_SYMMETRIC_TRANSFER + if (h.promise().next) { + return std::coroutine_handle<>::from_address(h.promise().next); + } +#else + void *loaded_ptr = h.promise().next; + if (loaded_ptr != nullptr) { + if (loaded_ptr == &h.promise().next) { + h.promise().next = nullptr; + return std::noop_coroutine(); + } + return std::coroutine_handle<>::from_address(loaded_ptr); + } +#endif + return std::noop_coroutine(); + } + + void await_resume() const noexcept {} + }; + + final_suspend_t final_suspend() noexcept { + return final_suspend_t{}; + } + + void unhandled_exception() { + exception = std::current_exception(); + } + + void *next = nullptr; + std::exception_ptr exception; + + static task_t get_return_object_on_allocation_failure() { + throw std::bad_alloc(); + } + + template + void *operator new(std::size_t n, [[maybe_unused]] Args &&...args) noexcept { + // todo:k2 think about args in new + // todo:k2 make coroutine allocator + void *buffer = get_platform_context()->allocator.alloc(n); + return buffer; + } + + void operator delete(void *ptr, size_t n) noexcept { + (void)n; + get_platform_context()->allocator.free(ptr); + } + }; + + template F> + struct promise_non_void_t : public promise_base_t { + template + requires std::constructible_from void return_value(E &&e) { + ::new (bytes) F(std::forward(e)); + } + + alignas(F) std::byte bytes[sizeof(F)]; + }; + + struct promise_void_t : public promise_base_t { + void return_void() {} + }; + + void execute() { + get_handle().resume(); + } + + T get_result() { + if (get_handle().promise().exception) { + std::rethrow_exception(std::move(get_handle().promise().exception)); + } + if constexpr (!std::is_void{}) { + T *t = std::launder(reinterpret_cast(get_handle().promise().bytes)); + const vk::final_action final_action([t] { t->~T(); }); + return std::move(*t); + } + } + + std::conditional_t{}, bool, std::optional> operator()() { + execute(); + if constexpr (std::is_void{}) { + get_result(); + return !done(); + } else { + if (!done()) { + return std::nullopt; + } + return get_result(); + } + } + + struct awaiter_t { + explicit awaiter_t(task_t *task) + : task{task} {} + + bool await_ready() const { + return false; + } + + template +#ifdef CPPCORO_COMPILER_SUPPORTS_SYMMETRIC_TRANSFER + std::coroutine_handle +#else + bool +#endif + await_suspend(std::coroutine_handle h) { +#ifdef CPPCORO_COMPILER_SUPPORTS_SYMMETRIC_TRANSFER + task->get_handle().promise().next = h.address(); + return task->get_handle(); +#else + void *next_ptr = &task->get_handle().promise().next; + task->get_handle().promise().next = next_ptr; + task->get_handle().resume(); + if (task->get_handle().promise().next == nullptr) { + return false; + } + task->get_handle().promise().next = h.address(); + return true; +#endif + } + + T await_resume() { + return task->get_result(); + } + + task_t *task; + }; + + awaiter_t operator co_await() { + return awaiter_t{this}; + } + + std::coroutine_handle get_handle() { + return std::coroutine_handle::from_address(handle_address); + } +}; diff --git a/runtime-light/header.h b/runtime-light/header.h new file mode 100644 index 0000000000..a5bfaeb645 --- /dev/null +++ b/runtime-light/header.h @@ -0,0 +1,279 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#ifndef K2_PLATFORM_HEADER_H +#define K2_PLATFORM_HEADER_H + +#pragma once + +#include + +#ifdef __cplusplus +#include +#include +#else +#include +#include +#endif + +#define K2_PLATFORM_HEADER_H_VERSION 5 + +// Always check that enum value is a valid value! + +#ifdef __cplusplus +extern "C" { +#endif + +struct Allocator { + // TODO zeroing or not? + void *(*alloc)(size_t); + void (*free)(void *); + void *(*realloc)(void *ptr, size_t); +}; + +enum IOStatus { + IOAvailable = 0, + IOBlocked = 1, + IOClosed = 2, +}; + +struct StreamStatus { + enum IOStatus read_status; + enum IOStatus write_status; + int32_t please_shutdown_write; +}; + +enum GetStatusResult { + GetStatusOk = 0, + GetStatusErrorNotStream = 1, + GetStatusErrorUseAfterFree = 2, + GetStatusErrorNoSuchStream = 3, +}; + +enum SetTimerResult { + SetTimerOk = 0, + SetTimerErrorLimitExceeded = 1, +}; + +enum TimerStatus { + TimerStatusScheduled = 0, + TimerStatusElapsed = 1, + TimerStatusInvalid = 2, +}; + +enum OpenStreamResult { + OpenStreamOk = 0, + // TODO: really need error? MB it's better to open and immediately close + // channel with corresponding error + OpenStreamErrorInvalidName = 1, + OpenStreamErrorUnknownComponent = 3, + OpenStreamErrorComponentUnavailable = 4, + OpenStreamErrorLimitExceeded = 5, +}; + +// This time point is valid only within the component. +// Similar to c++ `std::chrono::steady_clock::time_point` +struct TimePoint { + uint64_t time_point_ns; +}; + +struct SystemTime { + uint64_t since_epoch_ns; +}; + +#ifdef __cplusplus +typedef std::atomic_uint32_t atomic_uint32_t; +#else +typedef _Atomic(uint32_t) atomic_uint32_t; +#endif + +struct PlatformCtx { + atomic_uint32_t please_yield; + atomic_uint32_t please_graceful_shutdown; + + /* + * Immediately abort component execution. + * Function is [[noreturn]] + */ + void (*abort)(); + + struct Allocator allocator; + + /* + * In case of `result == Ok` stream_d will be NonZero + * + * In case of `result != Ok`: + * `stream_d` will be assigned `0`. + * however `stream_d=0` itself is not an error marker + */ + enum OpenStreamResult (*open)(size_t name_len, const char *name, + uint64_t *stream_d); + /* + * If the write or read status is `Blocked` - then the platform ensures that + * the component receives this `stream_d` via `take_update` when the status is + * no longer `Blocked` + * + * In case of `result != Ok`: + * `new_status` will be assigned as + * `{.read_status = 0, .write_status = 0, .please_shutdown = 0}`. + */ + enum GetStatusResult (*get_stream_status)(uint64_t stream_d, + struct StreamStatus *new_status); + /* + * Return processed bytes (written or read). + * Guaranteed to return `0` if the stream is `Closed`, `Blocked` or + * `stream_d` is invalid. + * May return `0` even if the stream is `Available` + * + * The bytes processed may be less than `data_len`, + * even if the stream is in `Available` state after the operation. + * Therefore, it does not indicate `Blocked` or `Closed` status. + * Use `get_stream_status` to check how to perform co-scheduling. + */ + size_t (*write)(uint64_t stream_d, size_t data_len, const void *data); + size_t (*read)(uint64_t stream_d, size_t data_len, void *data); + /* + * Sets `StreamStatus.please_whutdown_write=true` for the component on the + * opposite side (does not affect `StreamStatus` on your side). + * Does not disable the ability to read from the stream. + * To perform a graceful shutdown, you still need to read data from the stream + * as long as `read_status != IOClosed`. + */ + void (*please_shutdown_write)(uint64_t stream_d); + /* + * Disables the ability to write to a stream. + * Data written to the stream buffer is still available for reading on the + * opposite side. + * Does not affect the ability to read from the stream. + * + * TODO: design information errors. + */ + void (*shutdown_write)(uint64_t stream_d); + /* + * "Free" associated descriptor. + * All future uses of this `descriptor` will be invalid. + * + * In case were `descriptor` is stream: + * 1) If you are the owner of the stream (performed an `open`) - + * the read and write buffers will be immediately deleted. The opposite side + * will receive an `IOClosed` notification in both directions. + * 2) If you are not the owner, the opposite side will receive a notification + * `write_status = IOClosed`; Data sent from the opposite side and remaining + * in buffer will be deleted. The opposite side will be able to read the data + * sent from you and remaining in the buffer stream. + * + * In case where `descriptor` is timer: + * Ensures that no notification from this timer will be received. + * + * Noop in case if `descriptor` is invalid + */ + void (*free_descriptor)(uint64_t descriptor); + + // Time since epoch, non-monotonical + void (*get_system_time)(struct SystemTime *system_time); + + // Coordinated with timers. Monotonical, for timeouts, measurements, etc.. + void (*get_time)(struct TimePoint *time_point); + /* + * In case of `result == Ok` timer_d will be NonZero + * In case of `result != Ok` timer_d will be `0` + * + * One-shot timer + * Use `free_descriptor` to cancel + */ + enum SetTimerResult (*set_timer)(uint64_t *timer_d, uint64_t duration_ns); + /* + * It is guaranteed that if `TimerStatusElapsed` is returned + * then `deadline <= get_time()` + * There is no symmetric guarantee for `TimerStatusScheduled`. + * + * `deadline` will be assigned `0` if `timer_d` invalid + */ + enum TimerStatus (*get_timer_status)(uint64_t timer_d, + struct TimePoint *deadline); + /* + * Return: `bool`. + * If `True`: the update was successfully received. + * If `False`: no updates, `update_d` will be set to `0`. + * + * `update_d` might be either a stream or a timer. + * `update_d` can be a new stream installed for this component. + * Therefore, component should keep track of all active descriptors to + * distinguish new stream from existing one. + * + * `update_d` may be an elapsed timer. + * `get_timer_status` for this timer will be a valid call and will allow you + * to get its deadline (which is guaranteed to be in the past). + * But you must call `free_descriptor` to release the associated descriptor. + * + * If a component has not read all updates during a `poll` iteration, the + * platform is guaranteed to reschedule it. + */ + char (*take_update)(uint64_t *update_d); + /* + * Only utf-8 string supported. + * Possible `level` values: + * 1 => Error + * 2 => Warn + * 3 => Info + * 4 => Debug + * 5 => Trace + * Any other value will cause the log to be skipped + */ + void (*log)(size_t level, size_t len, const char *str); + /* + * if `level` > `log_level_enabled()` log will be skipped + */ + size_t (*log_level_enabled)(); +}; + +struct ComponentState; +/* + * Image state created once on library load + * shared between all component [instances]. + * designed to prevent heavy `_init` section of dlib + */ +struct ImageState; + +enum PollStatus { + // waitings IO or timer; no cpu work remains + PollBlocked = 0, + // there is some cpu work to do; platform will reschedule component + PollReschedule = 1, + // component decide to shutdown + PollFinishedOk = 2, + // component decide to shutdown + PollFinishedError = 3, +}; + +struct ImageInfo { + // TODO: null terminated string is OK? + // TODO: namespaces? + const char *image_name; + + uint64_t build_timestamp; + uint64_t header_h_version; + uint8_t commit_hash[40]; +}; + +// Every image should provide these symbols +enum PollStatus vk_k2_poll(const struct ImageState *image_state, + const struct PlatformCtx *pt_ctx, + struct ComponentState *component_ctx); + +// platform_ctx without IO stuff (nullptr instead io-functions) +// for now, returning nullptr will indicate error +struct ComponentState * +vk_k2_create_component_state(const struct ImageState *image_state, + const struct PlatformCtx *pt_ctx); + +// platform_ctx without IO stuff (nullptr instead io-functions) +// for now, returning nullptr will indicate error +struct ImageState *vk_k2_create_image_state(const struct PlatformCtx *pt_ctx); + +const struct ImageInfo *vk_k2_describe(); +#ifdef __cplusplus +} +#endif +#endif diff --git a/runtime-light/memory-resource-impl/monotonic-light-buffer-resource.cpp b/runtime-light/memory-resource-impl/monotonic-light-buffer-resource.cpp new file mode 100644 index 0000000000..d466778f7f --- /dev/null +++ b/runtime-light/memory-resource-impl/monotonic-light-buffer-resource.cpp @@ -0,0 +1,12 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-core/memory-resource/monotonic_buffer_resource.h" + +namespace memory_resource { +// todo:k2 How it should work in k2? +void monotonic_buffer_resource::critical_dump(void *, size_t) const noexcept {} + +void monotonic_buffer_resource::raise_oom(size_t) const noexcept {} +} // namespace memory_resource \ No newline at end of file diff --git a/runtime-light/runtime-light.cmake b/runtime-light/runtime-light.cmake new file mode 100644 index 0000000000..f0dd8f3305 --- /dev/null +++ b/runtime-light/runtime-light.cmake @@ -0,0 +1,60 @@ +include(${BASE_DIR}/runtime-light/allocator/allocator.cmake) +include(${BASE_DIR}/runtime-light/core/core.cmake) +include(${BASE_DIR}/runtime-light/stdlib/stdlib.cmake) +include(${BASE_DIR}/runtime-light/streams/streams.cmake) +include(${BASE_DIR}/runtime-light/utils/utils.cmake) + +prepend(MONOTOINC_LIGHT_BUFFER_RESOURCE_SRC ${BASE_DIR}/runtime-light/memory-resource-impl/ + monotonic-light-buffer-resource.cpp) + +prepend(RUNTIME_COMPONENT_SRC ${BASE_DIR}/runtime-light/ + component/component.cpp) + +set(RUNTIME_LIGHT_SRC ${RUNTIME_CORE_SRC} + ${RUNTIME_STDLIB_SRC} + ${RUNTIME_ALLOCATOR_SRC} + ${RUNTIME_COROUTINE_SRC} + ${RUNTIME_COMPONENT_SRC} + ${RUNTIME_STREAMS_SRC} + ${RUNTIME_UTILS_SRC} + ${RUNTIME_LANGUAGE_SRC} + ${MONOTOINC_LIGHT_BUFFER_RESOURCE_SRC} + ${BASE_DIR}/runtime-light/runtime-light.cpp) + +vk_add_library(runtime-light OBJECT ${RUNTIME_LIGHT_SRC}) +set_property(TARGET runtime-light PROPERTY POSITION_INDEPENDENT_CODE ON) +set_target_properties(runtime-light PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${BASE_DIR}/objs) + +vk_add_library(kphp-light-runtime STATIC) +target_link_libraries(kphp-light-runtime PUBLIC vk::light_common vk::runtime-light vk::runtime-core) +set_target_properties(kphp-light-runtime PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR}) + +file(GLOB_RECURSE KPHP_RUNTIME_ALL_HEADERS + RELATIVE ${BASE_DIR} + CONFIGURE_DEPENDS + "${BASE_DIR}/runtime-light/*.h") +list(TRANSFORM KPHP_RUNTIME_ALL_HEADERS REPLACE "^(.+)$" [[#include "\1"]]) +list(JOIN KPHP_RUNTIME_ALL_HEADERS "\n" MERGED_RUNTIME_HEADERS) +file(WRITE ${AUTO_DIR}/runtime/runtime-headers.h "\ +#ifndef MERGED_RUNTIME_LIGHT_HEADERS_H +#define MERGED_RUNTIME_LIGHT_HEADERS_H + +${MERGED_RUNTIME_HEADERS} + +#endif +") + +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/php_lib_version.cpp + [[ +#include "auto/runtime/runtime-headers.h" +]]) + +add_library(php_lib_version_j OBJECT ${CMAKE_CURRENT_BINARY_DIR}/php_lib_version.cpp) +target_compile_options(php_lib_version_j PRIVATE -I. -E) +add_dependencies(php_lib_version_j kphp-light-runtime) + +add_custom_command(OUTPUT ${OBJS_DIR}/php_lib_version.sha256 + COMMAND tail -n +3 $ | sha256sum | awk '{print $$1}' > ${OBJS_DIR}/php_lib_version.sha256 + DEPENDS php_lib_version_j $ + COMMENT "php_lib_version.sha256 generation") diff --git a/runtime-light/runtime-light.cpp b/runtime-light/runtime-light.cpp new file mode 100644 index 0000000000..9725c64520 --- /dev/null +++ b/runtime-light/runtime-light.cpp @@ -0,0 +1,72 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/component/component.h" +#include "runtime-light/component/image.h" +#include "runtime-light/core/globals/php-init-scripts.h" + +ImageState *vk_k2_create_image_state(const struct PlatformCtx *pt_ctx) { + // Note that in vk_k2_create_image_state available only allocator and logs from pt_ctx + platformCtx = pt_ctx; + php_debug("create image state on \"%s\"", vk_k2_describe()->image_name); + char *buffer = static_cast(platformCtx->allocator.alloc(sizeof(ImageState))); + if (buffer == nullptr) { + php_warning("cannot allocate enough memory for ImageState"); + return nullptr; + } + mutableImageState = new (buffer) ImageState(); + imageState = mutableImageState; + init_php_scripts_once_in_master(); + ImageState *mutable_image_state = mutableImageState; + php_debug("finish image state creation on \"%s\"", vk_k2_describe()->image_name); + reset_thread_locals(); + return mutable_image_state; +} + +ComponentState *vk_k2_create_component_state(const struct ImageState *image_state, const struct PlatformCtx *pt_ctx) { + // Note that in vk_k2_create_component_state available only allocator and logs from pt_ctx + imageState = image_state; + platformCtx = pt_ctx; + php_debug("create component state on \"%s\"", vk_k2_describe()->image_name); + char *buffer = static_cast(platformCtx->allocator.alloc(sizeof(ComponentState))); + if (buffer == nullptr) { + php_warning("cannot allocate enough memory for ComponentState"); + return nullptr; + } + componentState = new (buffer) ComponentState(); + componentState->init_script_execution(); + ComponentState *component_state = componentState; + php_debug("finish component state creation on \"%s\"", vk_k2_describe()->image_name); + reset_thread_locals(); + return component_state; +} + +PollStatus vk_k2_poll(const ImageState *image_state, const PlatformCtx *pt_ctx, ComponentState *component_ctx) { + imageState = image_state; + platformCtx = pt_ctx; + componentState = component_ctx; + + componentState->resume_if_was_rescheduled(); + uint64_t stream_d = 0; + while (platformCtx->take_update(&stream_d) && componentState->not_finished()) { + php_debug("take update on stream %lu", stream_d); + StreamStatus status; + GetStatusResult res = platformCtx->get_stream_status(stream_d, &status); + if (res != GetStatusOk) { + php_warning("get stream status %d", res); + } + php_debug("stream status %d, %d, %d", status.read_status, status.write_status, status.please_shutdown_write); + php_debug("opened_streams size %zu", componentState->opened_streams.size()); + if (componentState->is_stream_already_being_processed(stream_d)) { + php_debug("update on processed stream %lu", stream_d); + componentState->resume_if_wait_stream(stream_d, status); + } else { + componentState->process_new_input_stream(stream_d); + } + } + + PollStatus poll_status = componentState->poll_status; + reset_thread_locals(); + return poll_status; +} diff --git a/runtime-light/stdlib/interface.cpp b/runtime-light/stdlib/interface.cpp new file mode 100644 index 0000000000..f99c7ad954 --- /dev/null +++ b/runtime-light/stdlib/interface.cpp @@ -0,0 +1,19 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/interface.h" + +#include +#include +#include + +#include "runtime-core/runtime-core.h" +#include "runtime-light/component/component.h" +#include "runtime-light/coroutine/awaitable.h" + +int64_t f$rand() { + std::random_device rd; + int64_t dice_roll = rd(); + return dice_roll; +} diff --git a/runtime-light/stdlib/interface.h b/runtime-light/stdlib/interface.h new file mode 100644 index 0000000000..e527145578 --- /dev/null +++ b/runtime-light/stdlib/interface.h @@ -0,0 +1,15 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-core/runtime-core.h" +#include "runtime-light/coroutine/task.h" + +int64_t f$rand(); + +template +T f$make_clone(const T &x) { + return x; +} diff --git a/runtime-light/stdlib/misc.cpp b/runtime-light/stdlib/misc.cpp new file mode 100644 index 0000000000..aae4bca717 --- /dev/null +++ b/runtime-light/stdlib/misc.cpp @@ -0,0 +1,91 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/misc.h" + +#include "runtime-light/component/component.h" +#include "runtime-light/coroutine/awaitable.h" +#include "runtime-light/utils/json-functions.h" +#include "runtime-light/utils/panic.h" + +static int ob_merge_buffers() { + Response &response = get_component_context()->response; + php_assert(response.current_buffer >= 0); + int ob_first_not_empty = 0; + while (ob_first_not_empty < response.current_buffer && response.output_buffers[ob_first_not_empty].size() == 0) { + ob_first_not_empty++; + } + for (int i = ob_first_not_empty + 1; i <= response.current_buffer; i++) { + response.output_buffers[ob_first_not_empty].append(response.output_buffers[i].c_str(), response.output_buffers[i].size()); + } + return ob_first_not_empty; +} + +task_t parse_input_query(QueryType query_type) { + ComponentState &ctx = *get_component_context(); + php_assert(ctx.standard_stream == 0); + co_await wait_incoming_query_t{}; + ctx.standard_stream = ctx.incoming_pending_queries.front(); + ctx.incoming_pending_queries.pop_front(); + ctx.opened_streams[ctx.standard_stream] = StreamRuntimeStatus::NotBlocked; + + if (query_type == QueryType::HTTP) { + auto [buffer, size] = co_await read_all_from_stream(ctx.standard_stream); + init_http_superglobals(buffer, size); + get_platform_context()->allocator.free(buffer); + } else if (query_type == QueryType::COMPONENT) { + // Processing takes place in the calling function + } else { + php_critical_error("unexpected query type %d in parse_input_query", static_cast(query_type)); + } + co_return; +} + +task_t finish(int64_t exit_code, bool from_exit) { + (void)from_exit; + (void)exit_code; + // todo:k2 use exit_code + ComponentState &ctx = *get_component_context(); + if (ctx.standard_stream == 0) { + co_return; + } + int ob_total_buffer = ob_merge_buffers(); + Response &response = ctx.response; + auto &buffer = response.output_buffers[ob_total_buffer]; + bool ok = co_await write_all_to_stream(ctx.standard_stream, buffer.c_str(), buffer.size()); + if (!ok) { + php_warning("cannot write component result to input stream %lu", ctx.standard_stream); + } + free_all_descriptors(); + ctx.poll_status = PollStatus::PollFinishedOk; + co_return; +} + +task_t f$testyield() { + co_await test_yield_t{}; +} + +void f$check_shutdown() { + const PlatformCtx &ptx = *get_platform_context(); + if (get_platform_context()->please_graceful_shutdown.load()) { + php_notice("script was graceful shutdown"); + ptx.abort(); + } +} + +task_t f$exit(const mixed &v) { + if (v.is_string()) { + Response &response = get_component_context()->response; + response.output_buffers[response.current_buffer] << v; + co_await finish(0, true); + } else { + co_await finish(v.to_int(), true); + } + critical_error_handler(); +} + +void f$die([[maybe_unused]] const mixed &v) { + get_component_context()->poll_status = PollStatus::PollFinishedOk; + critical_error_handler(); +} \ No newline at end of file diff --git a/runtime-light/stdlib/misc.h b/runtime-light/stdlib/misc.h new file mode 100644 index 0000000000..6fabb051a5 --- /dev/null +++ b/runtime-light/stdlib/misc.h @@ -0,0 +1,22 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-core/runtime-core.h" +#include "runtime-light/coroutine/task.h" +#include "runtime-light/stdlib/superglobals.h" + +task_t f$testyield(); + +void f$check_shutdown(); + +task_t f$exit(const mixed &v = 0); + +void f$die(const mixed &v = 0); + +void reset(); + +task_t parse_input_query(QueryType query_type); +task_t finish(int64_t exit_code, bool from_exit); diff --git a/runtime-light/stdlib/output-control.cpp b/runtime-light/stdlib/output-control.cpp new file mode 100644 index 0000000000..8aa7eeb6aa --- /dev/null +++ b/runtime-light/stdlib/output-control.cpp @@ -0,0 +1,99 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/output-control.h" + +#include "runtime-light/component/component.h" +#include "runtime-light/stdlib/string-functions.h" + +static constexpr int system_level_buffer = 0; + +void f$ob_clean() { + Response &httpResponse = get_component_context()->response; + httpResponse.output_buffers[httpResponse.current_buffer].clean(); +} + +bool f$ob_end_clean() { + Response &httpResponse = get_component_context()->response; + if (httpResponse.current_buffer == system_level_buffer) { + return false; + } + + --httpResponse.current_buffer; + return true; +} + +Optional f$ob_get_clean() { + Response &httpResponse = get_component_context()->response; + if (httpResponse.current_buffer == system_level_buffer) { + return false; + } + + string result = httpResponse.output_buffers[httpResponse.current_buffer].str(); + return result; +} + +string f$ob_get_content() { + Response &httpResponse = get_component_context()->response; + return httpResponse.output_buffers[httpResponse.current_buffer].str(); +} + +void f$ob_start(const string &callback) { + Response &httpResponse = get_component_context()->response; + if (httpResponse.current_buffer + 1 == httpResponse.ob_max_buffers) { + php_warning("Maximum nested level of output buffering reached. Can't do ob_start(%s)", callback.c_str()); + return; + } + + if (!callback.empty()) { + php_critical_error("unsupported callback %s at buffering level %d", callback.c_str(), httpResponse.current_buffer + 1); + } + + ++httpResponse.current_buffer; +} + +void f$ob_flush() { + Response &httpResponse = get_component_context()->response; + if (httpResponse.current_buffer == 0) { + php_warning("ob_flush with no buffer opented"); + return; + } + --httpResponse.current_buffer; + print(httpResponse.output_buffers[httpResponse.current_buffer + 1]); + ++httpResponse.current_buffer; + f$ob_clean(); +} + +bool f$ob_end_flush() { + Response &httpResponse = get_component_context()->response; + if (httpResponse.current_buffer == 0) { + return false; + } + f$ob_flush(); + return f$ob_end_clean(); +} + +Optional f$ob_get_flush() { + Response &httpResponse = get_component_context()->response; + if (httpResponse.current_buffer == 0) { + return false; + } + string result = httpResponse.output_buffers[httpResponse.current_buffer].str(); + f$ob_flush(); + f$ob_end_clean(); + return result; +} + +Optional f$ob_get_length() { + Response &httpResponse = get_component_context()->response; + if (httpResponse.current_buffer == 0) { + return false; + } + return httpResponse.output_buffers[httpResponse.current_buffer].size(); +} + +int64_t f$ob_get_level() { + Response &httpResponse = get_component_context()->response; + return httpResponse.current_buffer; +} diff --git a/runtime-light/stdlib/output-control.h b/runtime-light/stdlib/output-control.h new file mode 100644 index 0000000000..d4a90aabc4 --- /dev/null +++ b/runtime-light/stdlib/output-control.h @@ -0,0 +1,33 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-core/runtime-core.h" + +struct Response { + int static constexpr ob_max_buffers = 50; + string_buffer output_buffers[ob_max_buffers]; + int current_buffer = 0; +}; + +void f$ob_clean(); + +bool f$ob_end_clean(); + +Optional f$ob_get_clean(); + +string f$ob_get_contents(); + +void f$ob_start(const string &callback = string()); + +void f$ob_flush(); + +bool f$ob_end_flush(); + +Optional f$ob_get_flush(); + +Optional f$ob_get_length(); + +int64_t f$ob_get_level(); diff --git a/runtime-light/stdlib/stdlib.cmake b/runtime-light/stdlib/stdlib.cmake new file mode 100644 index 0000000000..360b50ecf1 --- /dev/null +++ b/runtime-light/stdlib/stdlib.cmake @@ -0,0 +1,8 @@ +prepend(RUNTIME_STDLIB_SRC ${BASE_DIR}/runtime-light/stdlib/ + interface.cpp + misc.cpp + output-control.cpp + string-functions.cpp + variable-handling.cpp + superglobals.cpp +) diff --git a/runtime-light/stdlib/string-functions.cpp b/runtime-light/stdlib/string-functions.cpp new file mode 100644 index 0000000000..132c36281d --- /dev/null +++ b/runtime-light/stdlib/string-functions.cpp @@ -0,0 +1,101 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/string-functions.h" + +#include "runtime-light/component/component.h" + +void print(const char *s, size_t s_len) { + Response &response = get_component_context()->response; + response.output_buffers[response.current_buffer].append(s, s_len); +} + +void print(const char *s) { + print(s, strlen(s)); +} + +void print(const string_buffer &sb) { + print(sb.buffer(), sb.size()); +} + +void print(const string &s) { + print(s.c_str(), s.size()); +} + +void f$debug_print_string(const string &s) { + printf("debug_print_string ["); + for (int i = 0; i < s.size(); ++i) { + printf("%d, ", s.c_str()[i]); + } + printf("]\n"); +} + +Optional f$byte_to_int(const string &s) { + if (s.size() != 1) { + php_warning("Cannot convert non-byte string to int"); + return false; + } + return *s.c_str(); +} + +Optional f$int_to_byte(int64_t v) { + if (v > 127 || v < -128) { + php_warning("Cannot convert too big int to byte %ld", v); + return false; + } + char c = v; + string result(&c, 1); + return result; +} + +string str_concat(const string &s1, const string &s2) { + // for 2 argument concatenation it's not so uncommon to have at least one empty string argument; + // it happens in cases like `$prefix . $s` where $prefix could be empty depending on some condition + // real-world applications analysis shows that ~17.6% of all two arguments concatenations have + // at least one empty string argument + // + // checking both lengths for 0 is almost free, but when we step into those 17.6%, we get almost x10 + // faster concatenation and no heap allocations + // + // this idea is borrowed from the Go runtime + if (s1.empty()) { + return s2; + } + if (s2.empty()) { + return s1; + } + auto new_size = s1.size() + s2.size(); + return string(new_size, true).append_unsafe(s1).append_unsafe(s2).finish_append(); +} + +string str_concat(str_concat_arg s1, str_concat_arg s2) { + auto new_size = s1.size + s2.size; + return string(new_size, true).append_unsafe(s1.as_tmp_string()).append_unsafe(s2.as_tmp_string()).finish_append(); +} + +string str_concat(str_concat_arg s1, str_concat_arg s2, str_concat_arg s3) { + auto new_size = s1.size + s2.size + s3.size; + return string(new_size, true).append_unsafe(s1.as_tmp_string()).append_unsafe(s2.as_tmp_string()).append_unsafe(s3.as_tmp_string()).finish_append(); +} + +string str_concat(str_concat_arg s1, str_concat_arg s2, str_concat_arg s3, str_concat_arg s4) { + auto new_size = s1.size + s2.size + s3.size + s4.size; + return string(new_size, true) + .append_unsafe(s1.as_tmp_string()) + .append_unsafe(s2.as_tmp_string()) + .append_unsafe(s3.as_tmp_string()) + .append_unsafe(s4.as_tmp_string()) + .finish_append(); +} + +string str_concat(str_concat_arg s1, str_concat_arg s2, str_concat_arg s3, str_concat_arg s4, str_concat_arg s5) { + auto new_size = s1.size + s2.size + s3.size + s4.size + s5.size; + return string(new_size, true) + .append_unsafe(s1.as_tmp_string()) + .append_unsafe(s2.as_tmp_string()) + .append_unsafe(s3.as_tmp_string()) + .append_unsafe(s4.as_tmp_string()) + .append_unsafe(s5.as_tmp_string()) + .finish_append(); +} diff --git a/runtime-light/stdlib/string-functions.h b/runtime-light/stdlib/string-functions.h new file mode 100644 index 0000000000..c2ba29b38f --- /dev/null +++ b/runtime-light/stdlib/string-functions.h @@ -0,0 +1,74 @@ +#pragma once + +#include "runtime-core/runtime-core.h" +#include + +void print(const char *s, size_t s_len); + +void print(const char *s); + +void print(const string &s); + +void print(const string_buffer &sb); + +inline void f$echo(const string &s) { + print(s); +} + +inline int64_t f$print(const string &s) { + print(s); + return 1; +} + +inline int64_t f$strlen(const string &s) { + return s.size(); +} + +void f$debug_print_string(const string &s); + +string f$increment_byte(const string &s); + +Optional f$byte_to_int(const string &s); + +Optional f$int_to_byte(int64_t v); + +// str_concat_arg generalizes both tmp_string and string arguments; +// it can be constructed from both of them, so concat functions can operate +// on both tmp_string and string types +// there is a special (string, string) overloading for concat2 to +// allow the empty string result optimization to kick in +struct str_concat_arg { + const char *data; + string::size_type size; + + str_concat_arg(const string &s) + : data{s.c_str()} + , size{s.size()} {} + str_concat_arg(tmp_string s) + : data{s.data} + , size{s.size} {} + + tmp_string as_tmp_string() const noexcept { + return {data, size}; + } +}; + +// str_concat functions implement efficient string-typed `.` (concatenation) operator implementation; +// apart from being machine-code size efficient (a function call is more compact), they're also +// usually faster as runtime is compiled with -O3 which is almost never the case for translated C++ code +// (it's either -O2 or -Os most of the time) +// +// we choose to have 4 functions (up to 5 arguments) because of the frequency distribution: +// 37619: 2 args +// 20616: 3 args +// 4534: 5 args +// 3791: 4 args +// 935: 7 args +// 565: 6 args +// 350: 9 args +// Both 6 and 7 argument combination already look infrequent enough to not bother +string str_concat(const string &s1, const string &s2); +string str_concat(str_concat_arg s1, str_concat_arg s2); +string str_concat(str_concat_arg s1, str_concat_arg s2, str_concat_arg s3); +string str_concat(str_concat_arg s1, str_concat_arg s2, str_concat_arg s3, str_concat_arg s4); +string str_concat(str_concat_arg s1, str_concat_arg s2, str_concat_arg s3, str_concat_arg s4, str_concat_arg s5); diff --git a/runtime-light/stdlib/superglobals.cpp b/runtime-light/stdlib/superglobals.cpp new file mode 100644 index 0000000000..9ca0be6126 --- /dev/null +++ b/runtime-light/stdlib/superglobals.cpp @@ -0,0 +1,16 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/superglobals.h" + +#include "runtime-light/component/component.h" +#include "runtime-light/utils/json-functions.h" + +void init_http_superglobals(const char *buffer, int size) { + ComponentState &ctx = *get_component_context(); + string query = string(buffer, size); + mixed http = f$json_decode(query, true); + ctx.php_script_mutable_globals_singleton.get_superglobals().v$_SERVER.set_value(string("QUERY_TYPE"), string("http")); + ctx.php_script_mutable_globals_singleton.get_superglobals().v$_POST = http; +} diff --git a/runtime-light/stdlib/superglobals.h b/runtime-light/stdlib/superglobals.h new file mode 100644 index 0000000000..74ae2e802e --- /dev/null +++ b/runtime-light/stdlib/superglobals.h @@ -0,0 +1,12 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-core/runtime-core.h" +#include "runtime-light/coroutine/task.h" + +enum class QueryType { HTTP, COMPONENT }; + +void init_http_superglobals(const char *buffer, int size); diff --git a/runtime-light/stdlib/variable-handling.cpp b/runtime-light/stdlib/variable-handling.cpp new file mode 100644 index 0000000000..7b2ee44ab8 --- /dev/null +++ b/runtime-light/stdlib/variable-handling.cpp @@ -0,0 +1,211 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/variable-handling.h" + +#include "runtime-core/runtime-core.h" +#include "runtime-light/component/component.h" +#include "runtime-light/stdlib/output-control.h" +#include "runtime-light/utils/php_assert.h" + +void do_print_r(const mixed &v, int depth) { + if (depth == 10) { + php_warning("Depth %d reached. Recursion?", depth); + return; + } + Response &httpResponse = get_component_context()->response; + string_buffer *coub = &httpResponse.output_buffers[httpResponse.current_buffer]; + + switch (v.get_type()) { + case mixed::type::NUL: + break; + case mixed::type::BOOLEAN: + if (v.as_bool()) { + *coub << '1'; + } + break; + case mixed::type::INTEGER: + *coub << v.as_int(); + break; + case mixed::type::FLOAT: + *coub << v.as_double(); + break; + case mixed::type::STRING: + *coub << v.as_string(); + break; + case mixed::type::ARRAY: { + *coub << "Array\n"; + + string shift(depth << 3, ' '); + *coub << shift << "(\n"; + + for (array::const_iterator it = v.as_array().begin(); it != v.as_array().end(); ++it) { + *coub << shift << " [" << it.get_key() << "] => "; + do_print_r(it.get_value(), depth + 1); + *coub << '\n'; + } + + *coub << shift << ")\n"; + break; + } + default: + __builtin_unreachable(); + } +} + +static void do_var_dump(const mixed &v, int depth) { + if (depth == 10) { + php_warning("Depth %d reached. Recursion?", depth); + return; + } + + string shift(depth * 2, ' '); + Response &httpResponse = get_component_context()->response; + string_buffer *coub = &httpResponse.output_buffers[httpResponse.current_buffer]; + + switch (v.get_type()) { + case mixed::type::NUL: + *coub << shift << "NULL"; + break; + case mixed::type::BOOLEAN: + *coub << shift << "bool(" << (v.as_bool() ? "true" : "false") << ')'; + break; + case mixed::type::INTEGER: + *coub << shift << "int(" << v.as_int() << ')'; + break; + case mixed::type::FLOAT: + *coub << shift << "float(" << v.as_double() << ')'; + break; + case mixed::type::STRING: + *coub << shift << "string(" << (int)v.as_string().size() << ") \"" << v.as_string() << '"'; + break; + case mixed::type::ARRAY: { + *coub << shift << "array(" << v.as_array().count() << ") {\n"; + + for (array::const_iterator it = v.as_array().begin(); it != v.as_array().end(); ++it) { + *coub << shift << " ["; + if (array::is_int_key(it.get_key())) { + *coub << it.get_key(); + } else { + *coub << '"' << it.get_key() << '"'; + } + *coub << "]=>\n"; + do_var_dump(it.get_value(), depth + 1); + } + + *coub << shift << "}"; + break; + } + default: + __builtin_unreachable(); + } + *coub << '\n'; +} + +static void var_export_escaped_string(const string &s) { + Response &httpResponse = get_component_context()->response; + string_buffer *coub = &httpResponse.output_buffers[httpResponse.current_buffer]; + for (string::size_type i = 0; i < s.size(); i++) { + switch (s[i]) { + case '\'': + case '\\': + *coub << "\\" << s[i]; + break; + case '\0': + *coub << "\' . \"\\0\" . \'"; + break; + default: + *coub << s[i]; + } + } +} + +static void do_var_export(const mixed &v, int depth, char endc = 0) { + if (depth == 10) { + php_warning("Depth %d reached. Recursion?", depth); + return; + } + + string shift(depth * 2, ' '); + Response &httpResponse = get_component_context()->response; + string_buffer *coub = &httpResponse.output_buffers[httpResponse.current_buffer]; + + switch (v.get_type()) { + case mixed::type::NUL: + *coub << shift << "NULL"; + break; + case mixed::type::BOOLEAN: + *coub << shift << (v.as_bool() ? "true" : "false"); + break; + case mixed::type::INTEGER: + *coub << shift << v.as_int(); + break; + case mixed::type::FLOAT: + *coub << shift << v.as_double(); + break; + case mixed::type::STRING: + *coub << shift << '\''; + var_export_escaped_string(v.as_string()); + *coub << '\''; + break; + case mixed::type::ARRAY: { + const bool is_vector = v.as_array().is_vector(); + *coub << shift << "array(\n"; + + for (array::const_iterator it = v.as_array().begin(); it != v.as_array().end(); ++it) { + if (!is_vector) { + *coub << shift; + if (array::is_int_key(it.get_key())) { + *coub << it.get_key(); + } else { + *coub << '\'' << it.get_key() << '\''; + } + *coub << " =>"; + if (it.get_value().is_array()) { + *coub << "\n"; + do_var_export(it.get_value(), depth + 1, ','); + } else { + do_var_export(it.get_value(), 1, ','); + } + } else { + do_var_export(it.get_value(), depth + 1, ','); + } + } + + *coub << shift << ")"; + break; + } + default: + __builtin_unreachable(); + } + if (endc != 0) { + *coub << endc; + } + *coub << '\n'; +} + +string f$var_export(const mixed &v, bool buffered) { + if (buffered) { + f$ob_start(); + do_var_export(v, 0); + return f$ob_get_clean().val(); + } + do_var_export(v, 0); + return {}; +} + +string f$print_r(const mixed &v, bool buffered) { + if (buffered) { + f$ob_start(); + do_print_r(v, 0); + return f$ob_get_clean().val(); + } + + do_print_r(v, 0); + return {}; +} + +void f$var_dump(const mixed &v) { + do_var_dump(v, 0); +} \ No newline at end of file diff --git a/runtime-light/stdlib/variable-handling.h b/runtime-light/stdlib/variable-handling.h new file mode 100644 index 0000000000..1bf7c83ad8 --- /dev/null +++ b/runtime-light/stdlib/variable-handling.h @@ -0,0 +1,32 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-core/runtime-core.h" +#include "runtime-light/stdlib/string-functions.h" + +string f$print_r(const mixed &v, bool buffered = false); + +template +string f$print_r(const class_instance &v, bool buffered = false) { + php_warning("print_r used on object"); + return f$print_r(string(v.get_class(), (string::size_type)strlen(v.get_class())), buffered); +} + +string f$var_export(const mixed &v, bool buffered = false); + +template +string f$var_export(const class_instance &v, bool buffered = false) { + php_warning("print_r used on object"); + return f$var_export(string(v.get_class(), (string::size_type)strlen(v.get_class())), buffered); +} + +void f$var_dump(const mixed &v); + +template +void f$var_dump(const class_instance &v) { + php_warning("print_r used on object"); + return f$var_dump(string(v.get_class(), (string::size_type)strlen(v.get_class()))); +} diff --git a/runtime-light/streams/component-stream.cpp b/runtime-light/streams/component-stream.cpp new file mode 100644 index 0000000000..b7c9a757e1 --- /dev/null +++ b/runtime-light/streams/component-stream.cpp @@ -0,0 +1,47 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/streams/component-stream.h" + +bool f$ComponentStream$$is_read_closed(const class_instance &stream) { + StreamStatus status; + GetStatusResult res = get_platform_context()->get_stream_status(stream.get()->stream_d, &status); + if (res != GetStatusOk) { + php_warning("cannot get stream status error %d", res); + return true; + } + return status.read_status == IOClosed; +} + +bool f$ComponentStream$$is_write_closed(const class_instance &stream) { + StreamStatus status; + GetStatusResult res = get_platform_context()->get_stream_status(stream.get()->stream_d, &status); + if (res != GetStatusOk) { + php_warning("cannot get stream status error %d", res); + return true; + } + return status.write_status == IOClosed; +} + +bool f$ComponentStream$$is_please_shutdown_write(const class_instance &stream) { + StreamStatus status; + GetStatusResult res = get_platform_context()->get_stream_status(stream.get()->stream_d, &status); + if (res != GetStatusOk) { + php_warning("cannot get stream status error %d", res); + return true; + } + return status.please_shutdown_write; +} + +void f$ComponentStream$$close(const class_instance &stream) { + free_descriptor(stream->stream_d); +} + +void f$ComponentStream$$shutdown_write(const class_instance &stream) { + get_platform_context()->shutdown_write(stream->stream_d); +} + +void f$ComponentStream$$please_shutdown_write(const class_instance &stream) { + get_platform_context()->please_shutdown_write(stream->stream_d); +} diff --git a/runtime-light/streams/component-stream.h b/runtime-light/streams/component-stream.h new file mode 100644 index 0000000000..ee0f8b7b0d --- /dev/null +++ b/runtime-light/streams/component-stream.h @@ -0,0 +1,50 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "common/algorithms/hashes.h" +#include "common/wrappers/string_view.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-light/streams/streams.h" + +struct C$ComponentStream final : public refcountable_php_classes { + uint64_t stream_d{}; + + const char *get_class() const noexcept { + return "ComponentStream"; + } + + int32_t get_hash() const noexcept { + return static_cast(vk::std_hash(vk::string_view(C$ComponentStream::get_class()))); + } + + ~C$ComponentStream() { + free_descriptor(stream_d); + } +}; + +struct C$ComponentQuery final : public refcountable_php_classes { + uint64_t stream_d{}; + + const char *get_class() const noexcept { + return "ComponentQuery"; + } + + int32_t get_hash() const noexcept { + return static_cast(vk::std_hash(vk::string_view(C$ComponentQuery::get_class()))); + } + + ~C$ComponentQuery() { + free_descriptor(stream_d); + } +}; + +bool f$ComponentStream$$is_read_closed(const class_instance &stream); +bool f$ComponentStream$$is_write_closed(const class_instance &stream); +bool f$ComponentStream$$is_please_shutdown_write(const class_instance &stream); + +void f$ComponentStream$$close(const class_instance &stream); +void f$ComponentStream$$shutdown_write(const class_instance &stream); +void f$ComponentStream$$please_shutdown_write(const class_instance &stream); diff --git a/runtime-light/streams/interface.cpp b/runtime-light/streams/interface.cpp new file mode 100644 index 0000000000..66ecc70ae0 --- /dev/null +++ b/runtime-light/streams/interface.cpp @@ -0,0 +1,148 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/streams/interface.h" + +#include "runtime-light/component/component.h" +#include "runtime-light/coroutine/awaitable.h" +#include "runtime-light/stdlib/misc.h" +#include "runtime-light/streams/streams.h" + +task_t f$component_get_http_query() { + ComponentState &ctx = *get_component_context(); + if (ctx.standard_stream != 0) { + php_warning("previous incoming stream does not closed"); + ctx.standard_stream = 0; + } + co_await parse_input_query(QueryType::HTTP); +} + +task_t> f$component_client_send_query(const string &name, const string &message) { + class_instance query; + const PlatformCtx &ptx = *get_platform_context(); + uint64_t stream_d; + OpenStreamResult res = ptx.open(name.size(), name.c_str(), &stream_d); + if (res != OpenStreamOk) { + php_warning("cannot open stream"); + co_return query; + } + int writed = co_await write_all_to_stream(stream_d, message.c_str(), message.size()); + ptx.shutdown_write(stream_d); + php_debug("send %d bytes from %d to \"%s\" on stream %lu", writed, message.size(), name.c_str(), stream_d); + query.alloc(); + query.get()->stream_d = stream_d; + co_return query; +} + +task_t f$component_client_get_result(class_instance query) { + php_assert(!query.is_null()); + uint64_t stream_d = query.get()->stream_d; + if (stream_d == 0) { + php_warning("cannot get component client result"); + co_return string(); + } + + auto [buffer, size] = co_await read_all_from_stream(stream_d); + string result; + result.assign(buffer, size); + free_descriptor(stream_d); + query.get()->stream_d = 0; + php_debug("read %d bytes from stream %lu", size, stream_d); + co_return result; +} + +task_t f$component_server_send_result(const string &message) { + ComponentState &ctx = *get_component_context(); + bool ok = co_await write_all_to_stream(ctx.standard_stream, message.c_str(), message.size()); + if (!ok) { + php_warning("cannot send component result"); + } else { + php_debug("send result \"%s\"", message.c_str()); + } + free_descriptor(ctx.standard_stream); + ctx.standard_stream = 0; +} + +task_t f$component_server_get_query() { + ComponentState &ctx = *get_component_context(); + if (ctx.standard_stream != 0) { + ctx.standard_stream = 0; + } + co_await parse_input_query(QueryType::COMPONENT); + auto [buffer, size] = co_await read_all_from_stream(ctx.standard_stream); + string query = string(buffer, size); + get_platform_context()->allocator.free(buffer); + co_return query; +} + +task_t> f$component_accept_stream() { + ComponentState &ctx = *get_component_context(); + if (ctx.standard_stream != 0) { + php_warning("previous stream does not closed"); + free_descriptor(ctx.standard_stream); + ctx.standard_stream = 0; + } + co_await parse_input_query(QueryType::COMPONENT); + class_instance stream; + stream.alloc(); + stream.get()->stream_d = ctx.standard_stream; + co_return stream; +} + +class_instance f$component_open_stream(const string &name) { + class_instance query; + const PlatformCtx &ptx = *get_platform_context(); + ComponentState &ctx = *get_component_context(); + uint64_t stream_d; + OpenStreamResult res = ptx.open(name.size(), name.c_str(), &stream_d); + if (res != OpenStreamOk) { + php_warning("cannot open stream"); + return query; + } + ctx.opened_streams[stream_d] = StreamRuntimeStatus::NotBlocked; + query.alloc(); + query.get()->stream_d = stream_d; + php_debug("open stream %lu to %s", stream_d, name.c_str()); + return query; +} + +int64_t f$component_stream_write_nonblock(const class_instance &stream, const string &message) { + return write_nonblock_to_stream(stream.get()->stream_d, message.c_str(), message.size()); +} + +string f$component_stream_read_nonblock(const class_instance &stream) { + auto [ptr, size] = read_nonblock_from_stream(stream.get()->stream_d); + string result(ptr, size); + get_platform_context()->allocator.free(ptr); + return result; +} + +task_t f$component_stream_write_exact(const class_instance &stream, const string &message) { + int write = co_await write_exact_to_stream(stream->stream_d, message.c_str(), message.size()); + php_debug("write exact %d bytes to stream %lu", write, stream->stream_d); + co_return write; +} + +task_t f$component_stream_read_exact(const class_instance &stream, int64_t len) { + char *buffer = static_cast(RuntimeAllocator::current().alloc_script_memory(len)); + int read = co_await read_exact_from_stream(stream->stream_d, buffer, len); + string result(buffer, read); + RuntimeAllocator::current().free_script_memory(buffer, len); + php_debug("read exact %d bytes from stream %lu", read, stream->stream_d); + co_return result; +} + +void f$component_close_stream(const class_instance &stream) { + free_descriptor(stream->stream_d); +} + +void f$component_finish_stream_processing(const class_instance &stream) { + ComponentState &ctx = *get_component_context(); + if (stream->stream_d != ctx.standard_stream) { + php_warning("call server finish query on non server stream %lu", stream->stream_d); + return; + } + free_descriptor(ctx.standard_stream); + ctx.standard_stream = 0; +} diff --git a/runtime-light/streams/interface.h b/runtime-light/streams/interface.h new file mode 100644 index 0000000000..400b0aecda --- /dev/null +++ b/runtime-light/streams/interface.h @@ -0,0 +1,39 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-light/coroutine/task.h" +#include "runtime-light/streams/component-stream.h" + +constexpr int64_t v$COMPONENT_ERROR = -1; + +task_t f$component_get_http_query(); + +/** + * component query client blocked interface + * */ +task_t> f$component_client_send_query(const string &name, const string &message); +task_t f$component_client_get_result(class_instance query); + +/** + * component query server blocked interface + * */ +task_t f$component_server_get_query(); +task_t f$component_server_send_result(const string &message); + +/** + * component query low-level interface + * */ +class_instance f$component_open_stream(const string &name); +task_t> f$component_accept_stream(); + +int64_t f$component_stream_write_nonblock(const class_instance &stream, const string &message); +string f$component_stream_read_nonblock(const class_instance &stream); + +task_t f$component_stream_write_exact(const class_instance &stream, const string &message); +task_t f$component_stream_read_exact(const class_instance &stream, int64_t len); + +void f$component_close_stream(const class_instance &stream); +void f$component_finish_stream_processing(const class_instance &stream); diff --git a/runtime-light/streams/streams.cmake b/runtime-light/streams/streams.cmake new file mode 100644 index 0000000000..c08ade1fec --- /dev/null +++ b/runtime-light/streams/streams.cmake @@ -0,0 +1,5 @@ +prepend(RUNTIME_STREAMS_SRC ${BASE_DIR}/runtime-light/streams/ + interface.cpp + streams.cpp + component-stream.cpp +) diff --git a/runtime-light/streams/streams.cpp b/runtime-light/streams/streams.cpp new file mode 100644 index 0000000000..66d24bdb1d --- /dev/null +++ b/runtime-light/streams/streams.cpp @@ -0,0 +1,201 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/streams/streams.h" + +#include "runtime-light/component/component.h" +#include "runtime-light/coroutine/awaitable.h" +#include "runtime-light/utils/context.h" + +task_t> read_all_from_stream(uint64_t stream_d) { + co_await test_yield_t{}; + + constexpr int batch_size = 32; + const PlatformCtx &ptx = *get_platform_context(); + int buffer_capacity = batch_size; + char *buffer = static_cast(ptx.allocator.alloc(buffer_capacity)); + int buffer_size = 0; + StreamStatus status; + + do { + GetStatusResult res = ptx.get_stream_status(stream_d, &status); + if (res != GetStatusOk) { + php_warning("get stream status return status %d", res); + co_return std::make_pair(nullptr, 0); + } + + if (status.read_status == IOAvailable) { + if (buffer_capacity - buffer_size < batch_size) { + char *new_buffer = static_cast(ptx.allocator.alloc(buffer_capacity * 2)); + memcpy(new_buffer, buffer, buffer_size); + ptx.allocator.free(buffer); + buffer_capacity = buffer_capacity * 2; + buffer = new_buffer; + } + buffer_size += ptx.read(stream_d, batch_size, buffer + buffer_size); + } else if (status.read_status == IOBlocked) { + co_await read_blocked_t{stream_d}; + } + } while (status.read_status != IOClosed); + + co_return std::make_pair(buffer, buffer_size); +} + +std::pair read_nonblock_from_stream(uint64_t stream_d) { + constexpr int batch_size = 32; + const PlatformCtx &ptx = *get_platform_context(); + int buffer_capacity = batch_size; + char *buffer = static_cast(ptx.allocator.alloc(buffer_capacity)); + int buffer_size = 0; + StreamStatus status; + do { + GetStatusResult res = ptx.get_stream_status(stream_d, &status); + if (res != GetStatusOk) { + php_warning("get stream status return status %d", res); + return std::make_pair(nullptr, 0); + } + + if (status.read_status == IOAvailable) { + if (buffer_capacity - buffer_size < batch_size) { + char *new_buffer = static_cast(ptx.allocator.alloc(buffer_capacity * 2)); + memcpy(new_buffer, buffer, buffer_size); + ptx.allocator.free(buffer); + buffer_capacity = buffer_capacity * 2; + buffer = new_buffer; + } + buffer_size += ptx.read(stream_d, batch_size, buffer + buffer_size); + } else { + break; + } + } while (status.read_status != IOClosed); + + return std::make_pair(buffer, buffer_size); +} + +task_t read_exact_from_stream(uint64_t stream_d, char *buffer, int len) { + co_await test_yield_t{}; + + const PlatformCtx &ptx = *get_platform_context(); + int read = 0; + StreamStatus status{IOAvailable, IOAvailable, 0}; + + while (read != len && status.read_status != IOClosed) { + GetStatusResult res = ptx.get_stream_status(stream_d, &status); + if (res != GetStatusOk) { + php_warning("get stream status return status %d", res); + co_return 0; + } + + if (status.read_status == IOAvailable) { + read += ptx.read(stream_d, len - read, buffer + read); + } else if (status.read_status == IOBlocked) { + co_await read_blocked_t{stream_d}; + } else { + co_return read; + } + } + + co_return read; +} + +task_t write_all_to_stream(uint64_t stream_d, const char *buffer, int len) { + co_await test_yield_t{}; + + StreamStatus status; + const PlatformCtx &ptx = *get_platform_context(); + int writed = 0; + do { + GetStatusResult res = ptx.get_stream_status(stream_d, &status); + if (res != GetStatusOk) { + php_warning("get stream status return status %d", res); + co_return writed; + } + + if (status.please_shutdown_write) { + php_debug("stream %lu set please_shutdown_write. Stop writing", stream_d); + co_return writed; + } else if (status.write_status == IOAvailable) { + writed += ptx.write(stream_d, len - writed, buffer + writed); + } else if (status.write_status == IOBlocked) { + co_await write_blocked_t{stream_d}; + } else { + php_warning("stream closed while writing. Writed %d. Size %d. Stream %lu", writed, len, stream_d); + co_return writed; + } + } while (writed != len); + + php_debug("write %d bytes to stream %lu", len, stream_d); + co_return writed; +} + +int write_nonblock_to_stream(uint64_t stream_d, const char *buffer, int len) { + StreamStatus status; + const PlatformCtx &ptx = *get_platform_context(); + int writed = 0; + do { + GetStatusResult res = ptx.get_stream_status(stream_d, &status); + if (res != GetStatusOk) { + php_warning("get stream status return status %d", res); + return 0; + } + + if (status.write_status == IOAvailable) { + writed += ptx.write(stream_d, len - writed, buffer + writed); + } else { + break; + } + } while (writed != len); + + php_debug("write %d bytes from %d to stream %lu", writed, len, stream_d); + return writed; +} + +task_t write_exact_to_stream(uint64_t stream_d, const char *buffer, int len) { + co_await test_yield_t{}; + + StreamStatus status{IOAvailable, IOAvailable, 0}; + const PlatformCtx &ptx = *get_platform_context(); + int writed = 0; + while (writed != len && status.write_status != IOClosed) { + GetStatusResult res = ptx.get_stream_status(stream_d, &status); + if (res != GetStatusOk) { + php_warning("get stream status return status %d", res); + co_return writed; + } + + if (status.please_shutdown_write) { + php_debug("stream %lu set please_shutdown_write. Stop writing", stream_d); + co_return writed; + } else if (status.write_status == IOAvailable) { + writed += ptx.write(stream_d, len - writed, buffer + writed); + } else if (status.write_status == IOBlocked) { + co_await write_blocked_t{stream_d}; + } else { + co_return writed; + } + } + + co_return writed; +} + +void free_all_descriptors() { + php_debug("free all descriptors"); + ComponentState &ctx = *get_component_context(); + const PlatformCtx &ptx = *get_platform_context(); + for (auto &processed_query : ctx.opened_streams) { + ptx.free_descriptor(processed_query.first); + } + ctx.opened_streams.clear(); + ctx.awaiting_coroutines.clear(); + ptx.free_descriptor(ctx.standard_stream); + ctx.standard_stream = 0; +} + +void free_descriptor(uint64_t stream_d) { + php_debug("free descriptor %lu", stream_d); + ComponentState &ctx = *get_component_context(); + get_platform_context()->free_descriptor(stream_d); + ctx.opened_streams.erase(stream_d); + ctx.awaiting_coroutines.erase(stream_d); +} diff --git a/runtime-light/streams/streams.h b/runtime-light/streams/streams.h new file mode 100644 index 0000000000..692983ef44 --- /dev/null +++ b/runtime-light/streams/streams.h @@ -0,0 +1,25 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "runtime-core/runtime-core.h" + +#include "runtime-light/coroutine/task.h" + +enum class StreamRuntimeStatus { WBlocked, RBlocked, NotBlocked, Timer }; + +task_t> read_all_from_stream(uint64_t stream_d); +std::pair read_nonblock_from_stream(uint64_t stream_d); +task_t read_exact_from_stream(uint64_t stream_d, char *buffer, int len); + +task_t write_all_to_stream(uint64_t stream_d, const char *buffer, int len); +int write_nonblock_to_stream(uint64_t stream_d, const char *buffer, int len); +task_t write_exact_to_stream(uint64_t stream_d, const char *buffer, int len); + +void free_all_descriptors(); +void free_descriptor(uint64_t stream_d); diff --git a/runtime-light/utils/context.cpp b/runtime-light/utils/context.cpp new file mode 100644 index 0000000000..bba43de65a --- /dev/null +++ b/runtime-light/utils/context.cpp @@ -0,0 +1,17 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/utils/context.h" + +thread_local ImageState *mutableImageState; +const thread_local ImageState *imageState; +const thread_local PlatformCtx *platformCtx; +thread_local ComponentState *componentState; + +void reset_thread_locals() { + mutableImageState = nullptr; + imageState = nullptr; + platformCtx = nullptr; + componentState = nullptr; +} \ No newline at end of file diff --git a/runtime-light/utils/context.h b/runtime-light/utils/context.h new file mode 100644 index 0000000000..65b1bad7c6 --- /dev/null +++ b/runtime-light/utils/context.h @@ -0,0 +1,30 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-light/header.h" + +extern thread_local ImageState *mutableImageState; +extern const thread_local ImageState *imageState; +extern const thread_local PlatformCtx *platformCtx; +extern thread_local ComponentState *componentState; + +inline const PlatformCtx *get_platform_context() { + return platformCtx; +} + +inline ComponentState *get_component_context() { + return componentState; +} + +inline const ImageState *get_image_state() { + return imageState; +} + +inline ImageState *get_mutable_image_state() { + return mutableImageState; +} + +void reset_thread_locals(); diff --git a/runtime-light/utils/json-functions.cpp b/runtime-light/utils/json-functions.cpp new file mode 100644 index 0000000000..5fde8e0723 --- /dev/null +++ b/runtime-light/utils/json-functions.cpp @@ -0,0 +1,537 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/utils/json-functions.h" + +#include "common/algorithms/find.h" +#include "runtime-light/component/component.h" +// +//#include "runtime/string_functions.h" + +// note: json-functions.cpp is used for non-typed json implementation: for json_encode() and json_decode() +// for classes, e.g. `JsonEncoder::encode(new A)`, see json-writer.cpp and from/to visitors +namespace { + +void json_append_one_char(unsigned int c, string_buffer & sb) noexcept { + sb.append_char('\\'); + sb.append_char('u'); + sb.append_char("0123456789abcdef"[c >> 12]); + sb.append_char("0123456789abcdef"[(c >> 8) & 15]); + sb.append_char("0123456789abcdef"[(c >> 4) & 15]); + sb.append_char("0123456789abcdef"[c & 15]); +} + +bool json_append_char(unsigned int c, string_buffer & sb) noexcept { + if (c < 0x10000) { + if (0xD7FF < c && c < 0xE000) { + return false; + } + json_append_one_char(c, sb); + return true; + } + if (c <= 0x10ffff) { + c -= 0x10000; + json_append_one_char(0xD800 | (c >> 10), sb); + json_append_one_char(0xDC00 | (c & 0x3FF), sb); + return true; + } + return false; +} + + +bool do_json_encode_string_php(const JsonPath &json_path, const char *s, int len, int64_t options, string_buffer & sb) noexcept { + int begin_pos = sb.size(); + if (options & JSON_UNESCAPED_UNICODE) { + sb.reserve(2 * len + 2); + } else { + sb.reserve(6 * len + 2); + } + sb.append_char('"'); + + auto fire_error = [json_path, begin_pos, &sb](int pos) { + php_warning("%s: Not a valid utf-8 character at pos %d in function json_encode", json_path.to_string().c_str(), pos); + sb.set_pos(begin_pos); + sb.append("null", 4); + return false; + }; + + for (int pos = 0; pos < len; pos++) { + switch (s[pos]) { + case '"': + sb.append_char('\\'); + sb.append_char('"'); + break; + case '\\': + sb.append_char('\\'); + sb.append_char('\\'); + break; + case '/': + sb.append_char('\\'); + sb.append_char('/'); + break; + case '\b': + sb.append_char('\\'); + sb.append_char('b'); + break; + case '\f': + sb.append_char('\\'); + sb.append_char('f'); + break; + case '\n': + sb.append_char('\\'); + sb.append_char('n'); + break; + case '\r': + sb.append_char('\\'); + sb.append_char('r'); + break; + case '\t': + sb.append_char('\\'); + sb.append_char('t'); + break; + case 0 ... 7: + case 11: + case 14 ... 31: + json_append_one_char(s[pos], sb); + break; + case -128 ... - 1: { + const int a = s[pos]; + if ((a & 0x40) == 0) { + return fire_error(pos); + } + + const int b = s[++pos]; + if ((b & 0xc0) != 0x80) { + return fire_error(pos); + } + if ((a & 0x20) == 0) { + if ((a & 0x1e) <= 0) { + return fire_error(pos); + } + if (options & JSON_UNESCAPED_UNICODE) { + sb.append_char(static_cast(a)); + sb.append_char(static_cast(b)); + } else if (!json_append_char(((a & 0x1f) << 6) | (b & 0x3f), sb)) { + return fire_error(pos); + } + break; + } + + const int c = s[++pos]; + if ((c & 0xc0) != 0x80) { + return fire_error(pos); + } + if ((a & 0x10) == 0) { + if (((a & 0x0f) | (b & 0x20)) <= 0) { + return fire_error(pos); + } + if (options & JSON_UNESCAPED_UNICODE) { + sb.append_char(static_cast(a)); + sb.append_char(static_cast(b)); + sb.append_char(static_cast(c)); + } else if (!json_append_char(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f), sb)) { + return fire_error(pos); + } + break; + } + + const int d = s[++pos]; + if ((d & 0xc0) != 0x80) { + return fire_error(pos); + } + if ((a & 0x08) == 0) { + if (((a & 0x07) | (b & 0x30)) <= 0) { + return fire_error(pos); + } + if (options & JSON_UNESCAPED_UNICODE) { + sb.append_char(static_cast(a)); + sb.append_char(static_cast(b)); + sb.append_char(static_cast(c)); + sb.append_char(static_cast(d)); + } else if (!json_append_char(((a & 0x07) << 18) | ((b & 0x3f) << 12) | ((c & 0x3f) << 6) | (d & 0x3f), sb)) { + return fire_error(pos); + } + break; + } + + return fire_error(pos); + } + default: + sb.append_char(s[pos]); + break; + } + } + + sb.append_char('"'); + return true; +} + +} // namespace + +string JsonPath::to_string() const { + // this function is called only when error is occurred, so it's not + // very performance-sensitive + + if (depth == 0) { + return string{"/", 1}; + } + unsigned num_parts = std::clamp(depth, 0U, static_cast(arr.size())); + string result; + result.reserve_at_least((num_parts+1) * 8); + result.push_back('/'); + for (unsigned i = 0; i < num_parts; i++) { + const char *key = arr[i]; + if (key == nullptr) { + // int key indexing + result.append("[.]"); + } else { + // string key indexing + result.append("['"); + result.append(arr[i]); + result.append("']"); + } + } + if (depth >= arr.size()) { + result.append("..."); + } + return result; +} + +namespace impl_ { + +JsonEncoder::JsonEncoder(int64_t options, bool simple_encode, const char *json_obj_magic_key) noexcept: + options_(options), + simple_encode_(simple_encode), + json_obj_magic_key_(json_obj_magic_key) { +} + +bool JsonEncoder::encode(bool b, string_buffer & sb) noexcept { + if (b) { + sb.append("true", 4); + } else { + sb.append("false", 5); + } + return true; +} + +bool JsonEncoder::encode_null(string_buffer & sb) const noexcept { + sb.append("null", 4); + return true; +} + +bool JsonEncoder::encode(int64_t i, string_buffer & sb) noexcept { + sb << i; + return true; +} + +bool JsonEncoder::encode(double d, string_buffer & sb) noexcept { + if (vk::any_of_equal(std::fpclassify(d), FP_INFINITE, FP_NAN)) { + php_warning("%s: strange double %lf in function json_encode", json_path_.to_string().c_str(), d); + if (options_ & JSON_PARTIAL_OUTPUT_ON_ERROR) { + sb.append("0", 1); + } else { + return false; + } + } else { + //todo:k2 implement f$number_format + sb << /*(simple_encode_ ? f$number_format(d, 6, string{"."}, string{}) : */ string{d}/*)*/; + } + return true; +} + +bool JsonEncoder::encode(const string &s, string_buffer & sb) noexcept { + return do_json_encode_string_php(json_path_, s.c_str(), s.size(), options_, sb); +} + +bool JsonEncoder::encode(const mixed &v, string_buffer & sb) noexcept { + switch (v.get_type()) { + case mixed::type::NUL: + return encode_null(sb); + case mixed::type::BOOLEAN: + return encode(v.as_bool(), sb); + case mixed::type::INTEGER: + return encode(v.as_int(), sb); + case mixed::type::FLOAT: + return encode(v.as_double(), sb); + case mixed::type::STRING: + return encode(v.as_string(), sb); + case mixed::type::ARRAY: + return encode(v.as_array(), sb); + default: + __builtin_unreachable(); + } +} + +} // namespace impl_ + +namespace { + +void json_skip_blanks(const char *s, int &i) noexcept { + while (vk::any_of_equal(s[i], ' ', '\t', '\r', '\n')) { + i++; + } +} + +bool do_json_decode(const char *s, int s_len, int &i, mixed &v, const char *json_obj_magic_key) noexcept { + if (!v.is_null()) { + v.destroy(); + } + json_skip_blanks(s, i); + switch (s[i]) { + case 'n': + if (s[i + 1] == 'u' && + s[i + 2] == 'l' && + s[i + 3] == 'l') { + i += 4; + return true; + } + break; + case 't': + if (s[i + 1] == 'r' && + s[i + 2] == 'u' && + s[i + 3] == 'e') { + i += 4; + new(&v) mixed(true); + return true; + } + break; + case 'f': + if (s[i + 1] == 'a' && + s[i + 2] == 'l' && + s[i + 3] == 's' && + s[i + 4] == 'e') { + i += 5; + new(&v) mixed(false); + return true; + } + break; + case '"': { + int j = i + 1; + int slashes = 0; + while (j < s_len && s[j] != '"') { + if (s[j] == '\\') { + slashes++; + j++; + } + j++; + } + if (j < s_len) { + int len = j - i - 1 - slashes; + + string value(len, false); + + i++; + int l; + for (l = 0; l < len && i < j; l++) { + char c = s[i]; + if (c == '\\') { + i++; + switch (s[i]) { + case '"': + case '\\': + case '/': + value[l] = s[i]; + break; + case 'b': + value[l] = '\b'; + break; + case 'f': + value[l] = '\f'; + break; + case 'n': + value[l] = '\n'; + break; + case 'r': + value[l] = '\r'; + break; + case 't': + value[l] = '\t'; + break; + case 'u': + if (isxdigit(s[i + 1]) && isxdigit(s[i + 2]) && isxdigit(s[i + 3]) && isxdigit(s[i + 4])) { + int num = 0; + for (int t = 0; t < 4; t++) { + char c = s[++i]; + if ('0' <= c && c <= '9') { + num = num * 16 + c - '0'; + } else { + c |= 0x20; + if ('a' <= c && c <= 'f') { + num = num * 16 + c - 'a' + 10; + } + } + } + + if (0xD7FF < num && num < 0xE000) { + if (s[i + 1] == '\\' && s[i + 2] == 'u' && + isxdigit(s[i + 3]) && isxdigit(s[i + 4]) && isxdigit(s[i + 5]) && isxdigit(s[i + 6])) { + i += 2; + int u = 0; + for (int t = 0; t < 4; t++) { + char c = s[++i]; + if ('0' <= c && c <= '9') { + u = u * 16 + c - '0'; + } else { + c |= 0x20; + if ('a' <= c && c <= 'f') { + u = u * 16 + c - 'a' + 10; + } + } + } + + if (0xD7FF < u && u < 0xE000) { + num = (((num & 0x3FF) << 10) | (u & 0x3FF)) + 0x10000; + } else { + i -= 6; + return false; + } + } else { + return false; + } + } + + if (num < 128) { + value[l] = static_cast(num); + } else if (num < 0x800) { + value[l++] = static_cast(0xc0 + (num >> 6)); + value[l] = static_cast(0x80 + (num & 63)); + } else if (num < 0xffff) { + value[l++] = static_cast(0xe0 + (num >> 12)); + value[l++] = static_cast(0x80 + ((num >> 6) & 63)); + value[l] = static_cast(0x80 + (num & 63)); + } else { + value[l++] = static_cast(0xf0 + (num >> 18)); + value[l++] = static_cast(0x80 + ((num >> 12) & 63)); + value[l++] = static_cast(0x80 + ((num >> 6) & 63)); + value[l] = static_cast(0x80 + (num & 63)); + } + break; + } + /* fallthrough */ + default: + return false; + } + i++; + } else { + value[l] = s[i++]; + } + } + value.shrink(l); + + new(&v) mixed(value); + i++; + return true; + } + break; + } + case '[': { + array res; + i++; + json_skip_blanks(s, i); + if (s[i] != ']') { + do { + mixed value; + if (!do_json_decode(s, s_len, i, value, json_obj_magic_key)) { + return false; + } + res.push_back(value); + json_skip_blanks(s, i); + } while (s[i++] == ','); + + if (s[i - 1] != ']') { + return false; + } + } else { + i++; + } + + new(&v) mixed(res); + return true; + } + case '{': { + array res; + i++; + json_skip_blanks(s, i); + if (s[i] != '}') { + do { + mixed key; + if (!do_json_decode(s, s_len, i, key, json_obj_magic_key) || !key.is_string()) { + return false; + } + json_skip_blanks(s, i); + if (s[i++] != ':') { + return false; + } + + if (!do_json_decode(s, s_len, i, res[key], json_obj_magic_key)) { + return false; + } + json_skip_blanks(s, i); + } while (s[i++] == ','); + + if (s[i - 1] != '}') { + return false; + } + } else { + i++; + } + + // it's impossible to distinguish whether empty php array was an json array or json object; + // to overcome it we add dummy key to php array that make array::is_vector() returning false, so we have difference + if (json_obj_magic_key && res.empty()) { + res[string{json_obj_magic_key}] = true; + } + + new(&v) mixed(res); + return true; + } + default: { + int j = i; + while (s[j] == '-' || ('0' <= s[j] && s[j] <= '9') || s[j] == 'e' || s[j] == 'E' || s[j] == '+' || s[j] == '.') { + j++; + } + if (j > i) { + int64_t intval = 0; + if (php_try_to_int(s + i, j - i, &intval)) { + i = j; + new(&v) mixed(intval); + return true; + } + + char *end_ptr; + double floatval = strtod(s + i, &end_ptr); + if (end_ptr == s + j) { + i = j; + new(&v) mixed(floatval); + return true; + } + } + break; + } + } + + return false; +} + +} // namespace + +std::pair json_decode(const string &v, const char *json_obj_magic_key) noexcept { + mixed result; + int i = 0; + if (do_json_decode(v.c_str(), v.size(), i, result, json_obj_magic_key)) { + json_skip_blanks(v.c_str(), i); + if (i == static_cast(v.size())) { + bool success = true; + return {result, success}; + } + } + + return {}; +} + +mixed f$json_decode(const string &v, bool assoc) noexcept { + // TODO It was a warning before (in case if assoc is false), but then it was disabled, should we enable it again? + static_cast(assoc); + return json_decode(v).first; +} diff --git a/runtime-light/utils/json-functions.h b/runtime-light/utils/json-functions.h new file mode 100644 index 0000000000..15626a7e23 --- /dev/null +++ b/runtime-light/utils/json-functions.h @@ -0,0 +1,181 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" + + +constexpr int64_t JSON_UNESCAPED_UNICODE = 1; +constexpr int64_t JSON_FORCE_OBJECT = 16; +constexpr int64_t JSON_PRETTY_PRINT = 128; // TODO: add actual support to untyped +constexpr int64_t JSON_PARTIAL_OUTPUT_ON_ERROR = 512; +constexpr int64_t JSON_PRESERVE_ZERO_FRACTION = 1024; + +constexpr int64_t JSON_AVAILABLE_OPTIONS = JSON_UNESCAPED_UNICODE | JSON_FORCE_OBJECT | JSON_PARTIAL_OUTPUT_ON_ERROR; +constexpr int64_t JSON_AVAILABLE_FLAGS_TYPED = JSON_PRETTY_PRINT | JSON_PRESERVE_ZERO_FRACTION; + +struct JsonPath { + constexpr static int MAX_DEPTH = 8; + + std::array arr; + unsigned depth = 0; + + void enter(const char *key) noexcept { + if (depth < arr.size()) { + arr[depth] = key; + } + depth++; + } + + void leave() noexcept { + depth--; + } + + string to_string() const; +}; + +namespace impl_ { +// note: this class in runtime is used for non-typed json implementation: for json_encode() and json_decode() +// for classes, e.g. `JsonEncoder::encode(new A)`, see json-writer.h and from/to visitors +// todo somewhen, unify this JsonEncoder and JsonWriter, and support JSON_PRETTY_PRINT then +class JsonEncoder : vk::not_copyable { +public: + JsonEncoder(int64_t options, bool simple_encode, const char *json_obj_magic_key = nullptr) noexcept; + + //todo:k2 change static_SB everywhere to string_buffer arg + bool encode(bool b, string_buffer & sb) noexcept; + bool encode(int64_t i, string_buffer & sb) noexcept; + bool encode(const string &s, string_buffer & sb) noexcept; + bool encode(double d, string_buffer & sb) noexcept; + bool encode(const mixed &v, string_buffer & sb) noexcept; + + template + bool encode(const array &arr, string_buffer & sb) noexcept; + + template + bool encode(const Optional &opt, string_buffer & sb) noexcept; + +private: + bool encode_null(string_buffer & sb) const noexcept; + + JsonPath json_path_; + const int64_t options_{0}; + //todo:k2 use simple_encode + [[maybe_unused]] const bool simple_encode_{false}; + const char *json_obj_magic_key_{nullptr}; +}; + +template +bool JsonEncoder::encode(const array &arr, string_buffer & sb) noexcept { + bool is_vector = arr.is_vector(); + const bool force_object = static_cast(JSON_FORCE_OBJECT & options_); + if (!force_object && !is_vector && arr.is_pseudo_vector()) { + if (arr.get_next_key() == arr.count()) { + is_vector = true; + } else { + php_warning("%s: Corner case in json conversion, [] could be easy transformed to {}", json_path_.to_string().c_str()); + } + } + is_vector &= !force_object; + + sb << "{["[is_vector]; + + if (is_vector) { + int i = 0; + json_path_.enter(nullptr); // similar key for all entries + for (auto p : arr) { + if (i != 0) { + sb << ','; + } + if (!encode(p.get_value(), sb)) { + if (!(options_ & JSON_PARTIAL_OUTPUT_ON_ERROR)) { + return false; + } + } + i++; + } + json_path_.leave(); + } else { + bool is_first = true; + for (auto p : arr) { + if (!is_first) { + sb << ','; + } + is_first = false; + const char *next_key = nullptr; + const auto key = p.get_key(); + if (array::is_int_key(key)) { + auto int_key = key.to_int(); + next_key = nullptr; + sb << '"' << int_key << '"'; + } else { + const string &str_key = key.as_string(); + // skip service key intended only for distinguish empty json object with empty json array + if (json_obj_magic_key_ && !strcmp(json_obj_magic_key_, str_key.c_str())) { + continue; + } + next_key = str_key.c_str(); + if (!encode(str_key, sb)) { + if (!(options_ & JSON_PARTIAL_OUTPUT_ON_ERROR)) { + return false; + } + } + } + sb << ':'; + json_path_.enter(next_key); + if (!encode(p.get_value(), sb)) { + if (!(options_ & JSON_PARTIAL_OUTPUT_ON_ERROR)) { + return false; + } + } + json_path_.leave(); + } + } + + sb << "}]"[is_vector]; + return true; +} + +template +bool JsonEncoder::encode(const Optional &opt, string_buffer & sb) noexcept { + switch (opt.value_state()) { + case OptionalState::has_value: + return encode(opt.val(), sb); + case OptionalState::false_value: + return encode(false, sb); + case OptionalState::null_value: + return encode_null(sb); + } + __builtin_unreachable(); +} + +} // namespace impl_ + +template +Optional f$json_encode(const T &v, int64_t options = 0, bool simple_encode = false) noexcept { + const bool has_unsupported_option = static_cast(options & ~JSON_AVAILABLE_OPTIONS); + if (unlikely(has_unsupported_option)) { + php_warning("Wrong parameter options = %" PRIi64 " in function json_encode", options); + return false; + } + string_buffer sb; + if (unlikely(!impl_::JsonEncoder(options, simple_encode).encode(v, sb))) { + return false; + } + return sb.c_str(); +} + +//todo:k2 implement string f$vk_json_encode_safe(const T &v, bool simple_encode = true) noexcept + +template +inline Optional f$vk_json_encode(const T &v) noexcept { + return f$json_encode(v, 0, true); +} + +std::pair json_decode(const string &v, const char *json_obj_magic_key = nullptr) noexcept; +mixed f$json_decode(const string &v, bool assoc = false) noexcept; diff --git a/runtime-light/utils/logs.h b/runtime-light/utils/logs.h new file mode 100644 index 0000000000..fbc9c84561 --- /dev/null +++ b/runtime-light/utils/logs.h @@ -0,0 +1,13 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-light/header.h" + +#define Error 1 +#define Warn 2 +#define Info 3 +#define Debug 4 +#define Trace 5 diff --git a/runtime-light/utils/panic.h b/runtime-light/utils/panic.h new file mode 100644 index 0000000000..d11fbed624 --- /dev/null +++ b/runtime-light/utils/panic.h @@ -0,0 +1,24 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "context.h" +#include "runtime-light/component/component.h" +#include "runtime-light/utils/logs.h" + +inline void critical_error_handler() { + constexpr const char * message = "script panic"; + const PlatformCtx & ptx = *get_platform_context(); + ComponentState & ctx = *get_component_context(); + ptx.log(Debug, strlen(message), message); + + if (ctx.not_finished()) { + ctx.poll_status = PollStatus::PollFinishedError; + } + ptx.abort(); + exit(1); +} diff --git a/runtime-light/utils/php_assert.cpp b/runtime-light/utils/php_assert.cpp new file mode 100644 index 0000000000..0d869406bd --- /dev/null +++ b/runtime-light/utils/php_assert.cpp @@ -0,0 +1,65 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/utils/php_assert.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "runtime-light/utils/panic.h" + +static void php_warning_impl(bool out_of_memory, int error_type, char const *message, va_list args) { + (void)out_of_memory; + const int BUF_SIZE = 1000; + char buf[BUF_SIZE]; + + int size = vsnprintf(buf, BUF_SIZE, message, args); + get_platform_context()->log(error_type, size, buf); + if (error_type == Error) { + critical_error_handler(); + } +} + +void php_debug(char const *message, ...) { + va_list args; + va_start(args, message); + php_warning_impl(false, Debug, message, args); + va_end(args); +} + +void php_notice(char const *message, ...) { + va_list args; + va_start(args, message); + php_warning_impl(false, Info, message, args); + va_end(args); +} + +void php_warning(char const *message, ...) { + va_list args; + va_start(args, message); + php_warning_impl(false, Warn, message, args); + va_end(args); +} + +void php_error(char const *message, ...) { + va_list args; + va_start(args, message); + php_warning_impl(false, Error, message, args); + va_end(args); +} + +void php_assert__(const char *msg, const char *file, int line) { + php_error("Assertion \"%s\" failed in file %s on line %d", msg, file, line); + critical_error_handler(); + _exit(1); +} diff --git a/runtime-light/utils/php_assert.h b/runtime-light/utils/php_assert.h new file mode 100644 index 0000000000..41d30d4f7b --- /dev/null +++ b/runtime-light/utils/php_assert.h @@ -0,0 +1,13 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "common/wrappers/likely.h" +#include "common/mixin/not_copyable.h" + +#include "runtime-core/utils/kphp-assert-core.h" diff --git a/runtime-light/utils/timer.cpp b/runtime-light/utils/timer.cpp new file mode 100644 index 0000000000..3cb712012d --- /dev/null +++ b/runtime-light/utils/timer.cpp @@ -0,0 +1,21 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/utils/timer.h" + +void set_timer_impl(int64_t timeout_ms, on_timer_callback_t &&callback) { + const PlatformCtx &ptx = *get_platform_context(); + ComponentState &ctx = *get_component_context(); + uint64_t nanoseconds = static_cast(timeout_ms * 1e6); + uint64_t timer_d = 0; + SetTimerResult res = ptx.set_timer(&timer_d, nanoseconds); + if (res != SetTimerOk) { + php_warning("timer limit exceeded"); + return; + } + php_debug("set up timer %lu for %lu ms", timer_d, timeout_ms); + + ctx.opened_streams[timer_d] = StreamRuntimeStatus::Timer; + ctx.timer_callbacks[timer_d] = callback; +} \ No newline at end of file diff --git a/runtime-light/utils/timer.h b/runtime-light/utils/timer.h new file mode 100644 index 0000000000..e7ba0b5202 --- /dev/null +++ b/runtime-light/utils/timer.h @@ -0,0 +1,20 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-core/memory-resource/resource_allocator.h" +#include "runtime-light/component/component.h" + +// todo:k2 std::function use heap +using on_timer_callback_t = std::function; + +void set_timer_impl(int64_t timeout_ms, on_timer_callback_t &&callback); + +template +void f$set_timer(int64_t timeout, CallBack &&callback) { + set_timer_impl(timeout, on_timer_callback_t(std::forward(callback))); +} \ No newline at end of file diff --git a/runtime-light/utils/to-array-processor.h b/runtime-light/utils/to-array-processor.h new file mode 100644 index 0000000000..19ac809395 --- /dev/null +++ b/runtime-light/utils/to-array-processor.h @@ -0,0 +1,166 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/mixin/not_copyable.h" +#include "common/smart_ptrs/singleton.h" +#include "runtime-core/runtime-core.h" + +class ShapeKeyDemangle : vk::not_copyable { +public: + friend class vk::singleton; + + void init(std::unordered_map &&shape_keys_storage) noexcept { + inited_ = true; + shape_keys_storage_ = std::move(shape_keys_storage); + } + + std::string_view get_key_by(std::int64_t tag) const noexcept { + php_assert(inited_); + auto key_it = shape_keys_storage_.find(tag); + php_assert(key_it != shape_keys_storage_.end()); + return key_it->second; + } + +private: + ShapeKeyDemangle() = default; + + bool inited_{false}; + std::unordered_map shape_keys_storage_; +}; + +class ToArrayVisitor { +public: + explicit ToArrayVisitor(bool with_class_names) + : with_class_names_(with_class_names) {} + + array flush_result() && noexcept { + return std::move(result_); + } + + template + void operator()(const char *field_name, const T &value) { + process_impl(field_name, value); + } + + template + static void process_tuple(const std::tuple &tuple, ToArrayVisitor &visitor, std::index_sequence /*indexes*/) { + (visitor.process_impl("", std::get(tuple)), ...); + } + + template + static void process_shape(const shape, T...> &shape, ToArrayVisitor &visitor) { + auto &demangler = vk::singleton::get(); + (visitor.process_impl(demangler.get_key_by(Is).data(), shape.template get()), ...); + } + +private: + template + void process_impl(const char *field_name, const T &value) { + add_value(field_name, value); + } + + template + void process_impl(const char *field_name, const Optional &value) { + auto process_impl_lambda = [this, field_name](const auto &v) { return this->process_impl(field_name, v); }; + call_fun_on_optional_value(process_impl_lambda, value); + } + + template + void process_impl(const char *field_name, const array &value) { + array converted_value(value.size()); + for (auto it = value.begin(); it != value.end(); ++it) { + process_impl("", it.get_value()); + converted_value.set_value(it.get_key(), result_.pop()); + } + + add_value(field_name, converted_value); + } + + template + void process_impl(const char *field_name, const class_instance &instance) { + add_value(field_name, instance.is_null() ? mixed{} : f$to_array_debug(instance, with_class_names_)); + } + + template + void process_impl(const char *field_name, const std::tuple &value) { + ToArrayVisitor tuple_processor{with_class_names_}; + tuple_processor.result_.reserve(sizeof...(Args), true); + + process_tuple(value, tuple_processor, std::index_sequence_for{}); + add_value(field_name, std::move(tuple_processor).flush_result()); + } + + template + void process_impl(const char *field_name, const shape, T...> &value) { + ToArrayVisitor shape_processor{with_class_names_}; + shape_processor.result_.reserve(sizeof...(Is), true); + + process_shape(value, shape_processor); + add_value(field_name, std::move(shape_processor).flush_result()); + } + + template + void add_value(const char *field_name, T &&value) { + if (field_name[0] != '\0') { + result_.set_value(string{field_name}, std::forward(value)); + } else { + result_.push_back(std::forward(value)); + } + } + + array result_; + bool with_class_names_{false}; +}; + +template +array f$to_array_debug(const class_instance &klass, bool with_class_names = false) { + array result; + if (klass.is_null()) { + return result; + } + + if constexpr (!std::is_empty_v) { + ToArrayVisitor visitor{with_class_names}; + klass.get()->accept(visitor); + result = std::move(visitor).flush_result(); + } + + if (with_class_names) { + result.set_value(string("__class_name"), string(klass.get_class())); + } + return result; +} + +template +array f$to_array_debug(const std::tuple &tuple, bool with_class_names = false) { + ToArrayVisitor visitor{with_class_names}; + ToArrayVisitor::process_tuple(tuple, visitor, std::index_sequence_for{}); + return std::move(visitor).flush_result(); +} + +template +array f$to_array_debug(const shape, T...> &shape, bool with_class_names = false) { + ToArrayVisitor visitor{with_class_names}; + ToArrayVisitor::process_shape(shape, visitor); + return std::move(visitor).flush_result(); +} + +template +array f$instance_to_array(const class_instance &klass, bool with_class_names = false) { + return f$to_array_debug(klass, with_class_names); +} + +template +array f$instance_to_array(const std::tuple &tuple, bool with_class_names = false) { + return f$to_array_debug(tuple, with_class_names); +} + +template +array f$instance_to_array(const shape, T...> &shape, bool with_class_names = false) { + return f$to_array_debug(shape, with_class_names); +} diff --git a/runtime-light/utils/utils.cmake b/runtime-light/utils/utils.cmake new file mode 100644 index 0000000000..2106bb3bb7 --- /dev/null +++ b/runtime-light/utils/utils.cmake @@ -0,0 +1,5 @@ +prepend(RUNTIME_UTILS_SRC ${BASE_DIR}/runtime-light/utils/ + php_assert.cpp + json-functions.cpp + context.cpp + timer.cpp) diff --git a/runtime/context/runtime-core-allocator.cpp b/runtime/context/runtime-core-allocator.cpp index 143f02da17..83d661b217 100644 --- a/runtime/context/runtime-core-allocator.cpp +++ b/runtime/context/runtime-core-allocator.cpp @@ -38,6 +38,14 @@ void *RuntimeAllocator::alloc_global_memory(size_t size) noexcept { return dl::heap_allocate(size); } +void *RuntimeAllocator::alloc0_global_memory(size_t size) noexcept { + void * ptr = dl::heap_allocate(size); + if (ptr != nullptr) { + memset(ptr, 0, size); + } + return ptr; +} + void *RuntimeAllocator::realloc_global_memory(void *mem, size_t new_size, size_t old_size) noexcept { return dl::heap_reallocate(mem, new_size, old_size); } diff --git a/runtime/php_assert.cpp b/runtime/php_assert.cpp index 5b3b8a5d57..80decabd90 100644 --- a/runtime/php_assert.cpp +++ b/runtime/php_assert.cpp @@ -169,7 +169,7 @@ static void php_warning_impl(bool out_of_memory, int error_type, char const *mes } if (die_on_fail) { - raise_php_assert_signal__(); + critical_error_handler(); fprintf(stderr, "_exiting in php_warning, since such option is enabled\n"); _exit(1); } @@ -228,12 +228,12 @@ const char *php_uncaught_exception_error(const class_instance &ex) void php_assert__(const char *msg, const char *file, int line) { php_error("Assertion \"%s\" failed in file %s on line %d", msg, file, line); - raise_php_assert_signal__(); + critical_error_handler(); fprintf(stderr, "_exiting in php_assert\n"); _exit(1); } -void raise_php_assert_signal__() { +void critical_error_handler() { raise(SIGPHPASSERT); vk::singleton::get().fsync_log_file(); _exit(1); diff --git a/server/confdata-binlog-replay.cpp b/server/confdata-binlog-replay.cpp index 8ea8e8f396..d7efc8bebe 100644 --- a/server/confdata-binlog-replay.cpp +++ b/server/confdata-binlog-replay.cpp @@ -25,6 +25,10 @@ #include "common/server/init-snapshot.h" #include "common/tl/methods/string.h" #include "common/wrappers/string_view.h" +#include "common/kfs/kfs.h" + +#include "runtime-core/runtime-core.h" +#include "runtime/allocator.h" #include "runtime/confdata-global-manager.h" #include "runtime-core/runtime-core.h" #include "server/confdata-binlog-events.h" diff --git a/tests/k2-components/dev_null.php b/tests/k2-components/dev_null.php new file mode 100644 index 0000000000..99a368411c --- /dev/null +++ b/tests/k2-components/dev_null.php @@ -0,0 +1,3 @@ +is_write_closed() && !$stream_to_out->is_please_shutdown_write()) { + component_stream_write_exact($stream_to_out, "a"); +} + +component_close_stream($stream_to_out); +component_finish_stream_processing($input); + +$input = component_accept_stream(); + +while(!$input->is_write_closed() && !$input->is_please_shutdown_write()) { + component_stream_write_exact($input, "a"); +} + diff --git a/tests/k2-components/echo.php b/tests/k2-components/echo.php new file mode 100644 index 0000000000..c5f3cf1bf5 --- /dev/null +++ b/tests/k2-components/echo.php @@ -0,0 +1,12 @@ +stream = $s; + } + + public function close_stream() { + component_close_stream($this->stream); + } + + /** @param ComponentStream $stream*/ + public function reset($stream) { + $this->stream = $stream; + } +} + +class Reader extends StreamHolder { + public $count = 0; + + /** @param ComponentStream $s*/ + public function __construct($s) { + parent::__construct($s); + $this->count = 0; + } + + public function is_read_closed() { + return $this->stream->is_read_closed(); + } + + public function read_byte() { + $str = component_stream_read_exact($this->stream, 1); + $this->count++; + return $str; + } +} + +class Writer extends StreamHolder { + public $count = 0; + + /** @param ComponentStream $s*/ + public function __construct($s) { + parent::__construct($s); + $this->count = 0; + } + + public function is_write_closed() { + return $this->stream->is_write_closed(); + } + + public function write_exact($message) { + $this->count += component_stream_write_exact($this->stream, $message); + return; + } +} + +class BiDirectStream { + /** @var Reader $reader*/ + public $reader; + /** @var Writer $writer*/ + public $writer; + + public $name; + + public $last_saved_header; + + + /** @param ComponentStream $stream*/ + public function __construct($stream, $name) { + $this->reader = new Reader($stream); + $this->writer = new Writer($stream); + $this->name = $name; + } + + public function read_full_header() { + $m = $this->reader->read_byte(); + if ($m !== MARKER && !$this->reader->is_read_closed()) { + critical_error("here can be only marker, but got " . $m); + } + return $this->read_header(); + } + + public function read_header() { + $forward_id = byte_to_int($this->reader->read_byte()); + $seed = byte_to_int($this->reader->read_byte()); + if ($forward_id === null || $seed === null) { + return null; + } + $header = ["forward_id" => $forward_id, "seed" => $seed]; + return $header; + } + + public function get_last_header() { + return $this->last_saved_header; + } + + public function save_header($header) { + $this->last_saved_header = $header; + } + + public function write_full_header($header) { + $str = ""; + $str .= MARKER; + $str .= int_to_byte($header["forward_id"] + 1); + $str .= int_to_byte($header["seed"]); +// debug_print_string($str); + $this->writer->write_exact($str); + +// if ($this->writer->write_exact($str) != 3) { +// critical_error("cannot write header"); +// } + } + + public function reopen() { + $this->close_stream(); + $stream = component_open_stream($this->name); + if (is_null($stream)) { + + return false; + } + $this->reader->reset($stream); + $this->writer->reset($stream); + $this->reader->count = 0; + $this->writer->count = 0; + return true; + } + + public function close_stream() { + $this->reader->close_stream(); + } + + public function is_open() { + return !$this->reader->is_read_closed() && !$this->writer->is_write_closed(); + } + + public function get_stat() { + return [$this->reader->count, $this->writer->count]; + } +} + +/** + * @param BiDirectStream $prev + * @param BiDirectStream $next + */ +function print_stat($prev, $next) { + $a = $prev->get_stat(); + $b = $next->get_stat(); +} + +function setup_die_timer() { + global $scenario; + $type = $scenario["die"]["type"]; + if ($type === "NoDie") { + return; + } + $timeout = intval($scenario["die"]["time_ms"]); + set_timer($timeout, function() { + warning("die"); + die();}); +} + + +function get_destination($forward_id) +{ + global $scenario; + $len = count($scenario["forward"]["num"]); + return $scenario["forward"]["names"][0]; +} + +function check_forward_id($forward_id) { + global $scenario; + return intval($scenario["forward"]["num"][0]) === $forward_id; +} + +/** + * @param BiDirectStream $stream + */ +function echo_bytes($stream) { + while(!$stream->reader->is_read_closed() && !$stream->writer->is_write_closed()) { + backward_byte($stream, $stream); + } +} + +/** + * @param BiDirectStream $src + * @param BiDirectStream $dst + */ +function stream_header($src, $dst) { + $forward_header = $src->read_header(); + $dst->write_full_header($forward_header); + $dst->save_header($forward_header); + + $backward_header = $dst->read_full_header(); + if ($backward_header === null) { + return false; + } + $src->write_full_header($backward_header); + $src->save_header($backward_header); + return true; +} + +/** + * @param BiDirectStream $src + * @param BiDirectStream $dst + */ +function forward_byte($src, $dst) { + $str = $src->reader->read_byte(); + // cannot open stream to dst. Try reopen once + if ($dst->writer->is_write_closed()) { + $successful = $dst->reopen(); + if (!$successful) { + return false; + } + + if ($str !== MARKER) { + + // If there is no new message still need send last header + $last_header = $dst->get_last_header(); + $dst->write_full_header($last_header); + $header = $dst->read_full_header(); + if ($header === null) { + return false; + } + } + } + + if ($str === MARKER) { + $f = stream_header($src, $dst); + if (!$f) { + return false; + } + $str = $src->reader->read_byte(); + } + + $dst->writer->write_exact($str); + return true; +} + +/** + * @param BiDirectStream $src + * @param BiDirectStream $dst + */ +function backward_byte($src, $dst) { + $str = $src->reader->read_byte(); + if ($str === "") { + return false; + } + $dst->writer->write_exact($str); + return true; +} + + +/** + * @param BiDirectStream $prev + * @param BiDirectStream $next + */ +function stream_bytes($prev, $next) { + + //todo make header optional + + print_stat($prev, $next); + $header = $next->read_full_header(); + if ($next->reader->is_read_closed()) { + $successful = $next->reopen(); + if (!$successful) { + return; + } + $last_header = $next->get_last_header(); + $next->write_full_header($last_header); + $header = $next->read_full_header(); + } + + if ($next->reader->is_read_closed()) { + return; + } + + + $prev->write_full_header($header); + $prev->save_header($header); + print_stat($prev, $next); + + + while(true) { + if (!$prev->is_open()) { + break; + } + $ok = forward_byte($prev, $next); + if (!$prev->is_open() || !$ok) { + break; + } + $ok = backward_byte($next, $prev); + if (!$ok) { + break; + } + } +} + +/** + * @param ComponentStream $incoming_stream + */ +function process_query($incoming_stream) { + $incoming = new BiDirectStream($incoming_stream, "Unknown"); + $header = $incoming->read_full_header(); + $destination = get_destination($header["forward_id"]); + if ($incoming_stream->is_read_closed() || $incoming_stream->is_write_closed()) { + return; + } + + + + if ($destination === "echo") { + + $incoming->write_full_header($header); + echo_bytes($incoming); + return; + } + + $out = component_open_stream($destination); + if (is_null($out)) { + + return; + } + $outgoing = new BiDirectStream($out, $destination); + $outgoing->write_full_header($header); + $outgoing->save_header($header); + + stream_bytes($incoming, $outgoing); +} + +function main() { + while(true) { + + /** @var ComponentStream $incoming_stream */ + $incoming_stream = component_accept_stream(); + + if (is_null($incoming_stream)) { + continue; + } + + if(!$incoming_stream->is_read_closed() && !$incoming_stream->is_write_closed()) { + process_query($incoming_stream); + } + + component_finish_stream_processing($incoming_stream); + } +} + + + +$scenario_query = component_client_send_query("proptest-generator", ""); +if (is_null($scenario_query)) { + critical_error("proptest-generator cannot be null"); +} +$str = component_client_get_result($scenario_query); +$scenario = json_decode($str); +// warning(var_export($scenario, true)); +setup_die_timer(); + +main(); diff --git a/tests/k2-components/tight_loop.php b/tests/k2-components/tight_loop.php new file mode 100644 index 0000000000..f573330b03 --- /dev/null +++ b/tests/k2-components/tight_loop.php @@ -0,0 +1,6 @@ + Date: Wed, 26 Jun 2024 18:25:44 +0300 Subject: [PATCH 39/89] add support for runtime-light in compiler (#1026) --- compiler/code-gen/code-generator.h | 1 + compiler/code-gen/declarations.cpp | 6 +- compiler/code-gen/files/init-scripts.cpp | 96 ++++++++++++++++----- compiler/code-gen/files/init-scripts.h | 4 + compiler/code-gen/vertex-compiler.cpp | 14 ++- compiler/compiler-core.cpp | 2 + compiler/compiler-core.h | 5 ++ compiler/compiler-settings.cpp | 15 +++- compiler/compiler-settings.h | 1 + compiler/data/function-data.h | 1 + compiler/kphp2cpp.cpp | 8 +- compiler/make/make.cpp | 8 ++ compiler/make/objs-to-k2-component-target.h | 44 ++++++++++ compiler/pipes/calc-bad-vars.cpp | 17 ++++ compiler/pipes/calc-func-dep.cpp | 3 +- compiler/pipes/code-gen.cpp | 10 ++- compiler/pipes/parse-and-apply-phpdoc.cpp | 2 + 17 files changed, 204 insertions(+), 33 deletions(-) create mode 100644 compiler/make/objs-to-k2-component-target.h diff --git a/compiler/code-gen/code-generator.h b/compiler/code-gen/code-generator.h index d45cca6073..285aa3aef4 100644 --- a/compiler/code-gen/code-generator.h +++ b/compiler/code-gen/code-generator.h @@ -19,6 +19,7 @@ struct CGContext { std::vector catch_label_used; FunctionPtr parent_func; bool resumable_flag{false}; + bool interruptible_flag{false}; bool namespace_opened{false}; int inside_macro{0}; size_t inside_null_coalesce_fallback{0}; diff --git a/compiler/code-gen/declarations.cpp b/compiler/code-gen/declarations.cpp index 6692a35e23..31a3a97b6e 100644 --- a/compiler/code-gen/declarations.cpp +++ b/compiler/code-gen/declarations.cpp @@ -72,7 +72,11 @@ void FunctionDeclaration::compile(CodeGenerator &W) const { switch (style) { case gen_out_style::tagger: case gen_out_style::cpp: { - FunctionSignatureGenerator(W) << ret_type_gen << " " << FunctionName(function) << "(" << params_gen << ")"; + if (function->is_interruptible) { + FunctionSignatureGenerator(W) << "task_t<" << ret_type_gen << ">" << " " << FunctionName(function) << "(" << params_gen << ")"; + } else { + FunctionSignatureGenerator(W) << ret_type_gen << " " << FunctionName(function) << "(" << params_gen << ")"; + } break; } case gen_out_style::txt: { diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index a8d2d4fedb..8e6398c63d 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -40,25 +40,30 @@ void StaticInit::compile(CodeGenerator &W) const { W << "extern array gen$tl_storers_ht;" << NL; FunctionSignatureGenerator(W) << "void fill_tl_storers_ht()" << SemicolonAndNL() << NL; } - FunctionSignatureGenerator(W) << ("const char *get_php_scripts_version()") << BEGIN << "return " << RawString(G->settings().php_code_version.get()) << ";" - << NL << END << NL << NL; + if (!G->is_output_mode_k2_component()) { + FunctionSignatureGenerator(W) << ("const char *get_php_scripts_version()") << BEGIN << "return " << RawString(G->settings().php_code_version.get()) << ";" + << NL << END << NL << NL; + } - FunctionSignatureGenerator(W) << ("char **get_runtime_options([[maybe_unused]] int *count)") << BEGIN; - const auto &runtime_opts = G->get_kphp_runtime_opts(); - if (runtime_opts.empty()) { - W << "return nullptr;" << NL; - } else { - W << "*count = " << runtime_opts.size() << ";" << NL; - for (size_t i = 0; i != runtime_opts.size(); ++i) { - W << "static char arg" << i << "[] = " << RawString{runtime_opts[i]} << ";" << NL; - } - W << "static char *argv[] = " << BEGIN; - for (size_t i = 0; i != runtime_opts.size(); ++i) { - W << "arg" << i << "," << NL; + if (!G->is_output_mode_k2_component()) { + FunctionSignatureGenerator(W) << ("char **get_runtime_options([[maybe_unused]] int *count)") << BEGIN; + + const auto &runtime_opts = G->get_kphp_runtime_opts(); + if (runtime_opts.empty()) { + W << "return nullptr;" << NL; + } else { + W << "*count = " << runtime_opts.size() << ";" << NL; + for (size_t i = 0; i != runtime_opts.size(); ++i) { + W << "static char arg" << i << "[] = " << RawString{runtime_opts[i]} << ";" << NL; + } + W << "static char *argv[] = " << BEGIN; + for (size_t i = 0; i != runtime_opts.size(); ++i) { + W << "arg" << i << "," << NL; + } + W << END << ";" << NL << "return argv;" << NL; } - W << END << ";" << NL << "return argv;" << NL; + W << END << NL << NL; } - W << END << NL << NL; FunctionSignatureGenerator(W) << ("void init_php_scripts_once_in_master() ") << BEGIN; @@ -102,6 +107,20 @@ void StaticInit::compile(CodeGenerator &W) const { W << END << NL; } +struct RunInterruptedFunction { + FunctionPtr function; + RunInterruptedFunction(FunctionPtr function) : function(function) {} + + void compile(CodeGenerator &W) const { + std::string await_prefix = function->is_interruptible ? "co_await " : ""; + FunctionSignatureGenerator(W) << "task_t " << FunctionName(function) << "$run() " << BEGIN + << await_prefix << FunctionName(function) << "();" << NL + << "co_return;" + << END; + W << NL; + } +}; + struct RunFunction { FunctionPtr function; RunFunction(FunctionPtr function) : function(function) {} @@ -178,8 +197,11 @@ InitScriptsCpp::InitScriptsCpp(SrcFilePtr main_file_id) : void InitScriptsCpp::compile(CodeGenerator &W) const { W << OpenFile("init_php_scripts.cpp", "", false); - W << ExternInclude(G->settings().runtime_headers.get()) << - ExternInclude("server/php-init-scripts.h"); + W << ExternInclude(G->settings().runtime_headers.get()); + if (!G->is_output_mode_k2_component()) { + W << ExternInclude("server/php-init-scripts.h"); + } + W << Include(main_file_id->main_function->header_full_name); @@ -196,10 +218,18 @@ void InitScriptsCpp::compile(CodeGenerator &W) const { return; } - W << RunFunction(main_file_id->main_function) << NL; + if (G->is_output_mode_k2_component()) { + W << RunInterruptedFunction(main_file_id->main_function) << NL; + } else { + W << RunFunction(main_file_id->main_function) << NL; + } W << GlobalsResetFunction(main_file_id->main_function) << NL; - FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + if (G->is_output_mode_k2_component()) { + FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() << ", task_t&run" ")" << BEGIN; + } else { + FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + } W << "global_vars_allocate(php_globals);" << NL; for (LibPtr lib: G->get_libs()) { @@ -211,9 +241,13 @@ void InitScriptsCpp::compile(CodeGenerator &W) const { W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL; - W << "set_script (" - << FunctionName(main_file_id->main_function) << "$run, " - << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; + if (G->is_output_mode_k2_component()) { + W << "run = " << FunctionName(main_file_id->main_function) << "$run();" << NL; + } else { + W << "set_script (" + << FunctionName(main_file_id->main_function) << "$run, " + << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; + } W << END; @@ -239,3 +273,19 @@ void CppMainFile::compile(CodeGenerator &W) const { << END; W << CloseFile(); } + +void ComponentInfoFile::compile(CodeGenerator &W) const { + kphp_assert(G->is_output_mode_k2_component()); + G->settings().get_version(); + auto now = std::chrono::system_clock::now(); + W << OpenFile("image_info.cpp"); + W << ExternInclude(G->settings().runtime_headers.get()); + W << "const ImageInfo *vk_k2_describe() " << BEGIN + << "static ImageInfo imageInfo {\"" << G->settings().k2_component_name.get() << "\"" << "," + << std::chrono::duration_cast(now.time_since_epoch()).count() << "," + << "K2_PLATFORM_HEADER_H_VERSION, " + << "{" << "}};" << NL //todo:k2 add commit hash + << "return &imageInfo;" << NL + << END; + W << CloseFile(); +} diff --git a/compiler/code-gen/files/init-scripts.h b/compiler/code-gen/files/init-scripts.h index 80bef5b327..1c476fbf7c 100644 --- a/compiler/code-gen/files/init-scripts.h +++ b/compiler/code-gen/files/init-scripts.h @@ -23,3 +23,7 @@ struct LibVersionHFile : CodeGenRootCmd { struct CppMainFile : CodeGenRootCmd { void compile(CodeGenerator &W) const final; }; + +struct ComponentInfoFile : CodeGenRootCmd { + void compile(CodeGenerator &W) const final; +}; diff --git a/compiler/code-gen/vertex-compiler.cpp b/compiler/code-gen/vertex-compiler.cpp index d75ce37105..71ed0feed1 100644 --- a/compiler/code-gen/vertex-compiler.cpp +++ b/compiler/code-gen/vertex-compiler.cpp @@ -631,6 +631,7 @@ void compile_do(VertexAdaptor root, CodeGenerator &W) { void compile_return(VertexAdaptor root, CodeGenerator &W) { bool resumable_flag = W.get_context().resumable_flag; + bool interruptible_flag = W.get_context().interruptible_flag; if (resumable_flag) { if (root->has_expr()) { W << "RETURN " << MacroBegin{}; @@ -638,7 +639,11 @@ void compile_return(VertexAdaptor root, CodeGenerator &W) { W << "RETURN_VOID " << MacroBegin{}; } } else { - W << "return "; + if (interruptible_flag) { + W << "co_return "; + } else { + W << "return "; + } } if (root->has_expr()) { @@ -841,6 +846,9 @@ void compile_func_call(VertexAdaptor root, CodeGenerator &W, func_ if (mode == func_call_mode::fork_call) { W << FunctionForkName(func); } else { + if (func->is_interruptible) { + W << "(" << "co_await "; + } W << FunctionName(func); } } @@ -865,6 +873,9 @@ void compile_func_call(VertexAdaptor root, CodeGenerator &W, func_ W << JoinValues(args, ", "); W << ")"; + if (func->is_interruptible) { + W << ")"; + } } void compile_func_call_fast(VertexAdaptor root, CodeGenerator &W) { @@ -1448,6 +1459,7 @@ void compile_function(VertexAdaptor func_root, CodeGenerator &W) { W.get_context().parent_func = func; W.get_context().resumable_flag = func->is_resumable; + W.get_context().interruptible_flag = func->is_interruptible; if (func->is_resumable) { compile_function_resumable(func_root, W); diff --git a/compiler/compiler-core.cpp b/compiler/compiler-core.cpp index 014d04ec89..0c1fee7386 100644 --- a/compiler/compiler-core.cpp +++ b/compiler/compiler-core.cpp @@ -126,6 +126,8 @@ void CompilerCore::register_settings(CompilerSettings *settings) { output_mode = OutputMode::cli; } else if (settings->mode.get() == "lib") { output_mode = OutputMode::lib; + } else if (settings->mode.get() == "k2-component") { + output_mode = OutputMode::k2_component; } else { output_mode = OutputMode::server; } diff --git a/compiler/compiler-core.h b/compiler/compiler-core.h index 944f0fec47..e3fd2dd805 100644 --- a/compiler/compiler-core.h +++ b/compiler/compiler-core.h @@ -28,6 +28,7 @@ enum class OutputMode { server, // -M server cli, // -M cli lib, // -M lib + k2_component // -M k2-component }; class CompilerCore { @@ -176,6 +177,10 @@ class CompilerCore { return output_mode == OutputMode::lib; } + bool is_output_mode_k2_component() const { + return output_mode == OutputMode::k2_component; + } + Stats stats; }; diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index 67560eeca1..b17917bab5 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -216,6 +216,13 @@ void CompilerSettings::init() { option_as_dir(kphp_src_path); functions_file.value_ = get_full_path(functions_file.get()); runtime_sha256_file.value_ = get_full_path(runtime_sha256_file.get()); + if (link_file.value_.empty()) { + if (mode.get() == "k2-component") { + link_file.value_ = kphp_src_path.get() + "/objs/libkphp-light-runtime.a"; + } else { + link_file.value_ = kphp_src_path.get() + "/objs/libkphp-full-runtime.a"; + } + } link_file.value_ = get_full_path(link_file.get()); if (mode.get() == "lib") { @@ -283,7 +290,7 @@ void CompilerSettings::init() { if (!no_pch.get()) { ss << " -Winvalid-pch -fpch-preprocess"; } - if (dynamic_incremental_linkage.get()) { + if (mode.get() == "k2-component" || dynamic_incremental_linkage.get()) { ss << " -fPIC"; } if (vk::contains(cxx.get(), "clang")) { @@ -386,7 +393,11 @@ void CompilerSettings::init() { option_as_dir(dest_dir); dest_cpp_dir.value_ = dest_dir.get() + "kphp/"; dest_objs_dir.value_ = dest_dir.get() + "objs/"; - binary_path.value_ = dest_dir.get() + mode.get(); + if (mode.get() == "k2-component") { + binary_path.value_ = dest_dir.get() + k2_component_name.get() + ".so"; + } else { + binary_path.value_ = dest_dir.get() + mode.get(); + } performance_analyze_report_path.value_ = dest_dir.get() + "performance_issues.json"; generated_runtime_path.value_ = kphp_src_path.get() + "objs/generated/auto/runtime/"; diff --git a/compiler/compiler-settings.h b/compiler/compiler-settings.h index dc00a23c08..f1cd813bb2 100644 --- a/compiler/compiler-settings.h +++ b/compiler/compiler-settings.h @@ -147,6 +147,7 @@ class CompilerSettings : vk::not_copyable { KphpOption compilation_metrics_file; KphpOption override_kphp_version; KphpOption php_code_version; + KphpOption k2_component_name; KphpOption cxx; KphpOption cxx_toolchain_dir; diff --git a/compiler/data/function-data.h b/compiler/data/function-data.h index bfe1139501..3962e49b62 100644 --- a/compiler/data/function-data.h +++ b/compiler/data/function-data.h @@ -117,6 +117,7 @@ class FunctionData { bool cpp_template_call = false; bool cpp_variadic_call = false; bool is_resumable = false; + bool is_interruptible = false; bool can_be_implicitly_interrupted_by_other_resumable = false; bool is_virtual_method = false; bool is_overridden_method = false; diff --git a/compiler/kphp2cpp.cpp b/compiler/kphp2cpp.cpp index 33589f832e..5eadfe6c27 100644 --- a/compiler/kphp2cpp.cpp +++ b/compiler/kphp2cpp.cpp @@ -215,10 +215,10 @@ int main(int argc, char *argv[]) { 'f', "functions-file", "KPHP_FUNCTIONS", "${KPHP_PATH}/builtin-functions/kphp-full/_functions.txt"); parser.add("File with kphp runtime sha256 hash", settings->runtime_sha256_file, "runtime-sha256", "KPHP_RUNTIME_SHA256", "${KPHP_PATH}/objs/php_lib_version.sha256"); - parser.add("The output binary type: server, cli or lib", settings->mode, - 'M', "mode", "KPHP_MODE", "server", {"server", "cli", "lib"}); + parser.add("The output binary type: server, k2-component, cli or lib", settings->mode, + 'M', "mode", "KPHP_MODE", "server", {"server", "k2-component", "cli", "lib"}); parser.add("A runtime library for building the output binary", settings->link_file, - 'l', "link-with", "KPHP_LINK_FILE", "${KPHP_PATH}/objs/libkphp-full-runtime.a"); + 'l', "link-with", "KPHP_LINK_FILE"); parser.add("Directory where php files will be searched", settings->includes, 'I', "include-dir", "KPHP_INCLUDE_DIR"); parser.add("Destination directory", settings->dest_dir, @@ -293,6 +293,8 @@ int main(int argc, char *argv[]) { "require-functions-typing", "KPHP_REQUIRE_FUNCTIONS_TYPING"); parser.add("Require class typing (1 - @var / default value is mandatory, 0 - auto infer or check if exists)", settings->require_class_typing, "require-class-typing", "KPHP_REQUIRE_CLASS_TYPING"); + parser.add("Define k2 component name. Default is \"KPHP\"", settings->k2_component_name, + "k2-component-name", "KPHP_K2_COMPONENT_NAME", "KPHP"); parser.add_implicit_option("Linker flags", settings->ld_flags); parser.add_implicit_option("Incremental linker flags", settings->incremental_linker_flags); diff --git a/compiler/make/make.cpp b/compiler/make/make.cpp index 988d910585..2728dfd59e 100644 --- a/compiler/make/make.cpp +++ b/compiler/make/make.cpp @@ -22,6 +22,7 @@ #include "compiler/make/make-runner.h" #include "compiler/make/objs-to-bin-target.h" #include "compiler/make/objs-to-obj-target.h" +#include "compiler/make/objs-to-k2-component-target.h" #include "compiler/make/objs-to-static-lib-target.h" #include "compiler/stage.h" #include "compiler/threading/profiler.h" @@ -98,6 +99,10 @@ class MakeSetup { return create_target(new Objs2StaticLibTarget, to_targets(std::move(objs)), lib); } + Target *create_objs2k2_component_target(std::vector objs, File *lib) { + return create_target(new Objs2K2ComponentTarget, to_targets(std::move(objs)), lib); + } + bool make_target(File *bin, const std::string &build_message, int jobs_count) { return make.make_targets(to_targets(bin), build_message, jobs_count); } @@ -177,6 +182,7 @@ static long long get_imported_header_mtime(const std::string &header_path, const return 0; } + // prepare dir kphp_out/objs/pch_{flags} and make a target runtime-headers.h.gch inside it // in production, there will be two pch_ folders: with debug symbols and without them // later on, we'll copy this folder inside /tmp/kphp_pch — that's why if it exists, don't do anything @@ -431,6 +437,8 @@ void run_make() { if (output_mode == OutputMode::lib) { make.create_objs2static_lib_target(objs, &bin_file); + } else if (output_mode == OutputMode::k2_component) { + make.create_objs2k2_component_target(objs, &bin_file); } else { const std::string build_stage{"Compiling"}; AutoProfiler profiler{get_profiler(build_stage)}; diff --git a/compiler/make/objs-to-k2-component-target.h b/compiler/make/objs-to-k2-component-target.h new file mode 100644 index 0000000000..9c8cf75c43 --- /dev/null +++ b/compiler/make/objs-to-k2-component-target.h @@ -0,0 +1,44 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "compiler/compiler-settings.h" +#include "compiler/make/target.h" + +class Objs2K2ComponentTarget : public Target { + static std::string load_all_symbols_pre() { +#if defined(__APPLE__) + return "-Wl,-force_load "; +#else + return "-Wl,--whole-archive "; +#endif + } + + static std::string load_all_symbols_post() { +#if defined(__APPLE__) + return " "; +#else + return " -Wl,--no-whole-archive "; +#endif + } + +public: + std::string get_cmd() final { + std::stringstream ss; + ss << settings->cxx.get() << " -shared -o " << target() << " "; + + for (size_t i = 0; i + 1 < deps.size(); ++i) { + ss << deps[i]->get_name() << " "; + } + + // the last dep is runtime lib + // todo:k2 think about kphp-libraries + assert(deps.size() >= 1 && "There are should be at least one dependency. It's the runtime lib"); + ss << load_all_symbols_pre() << deps.back()->get_name() << load_all_symbols_post(); + return ss.str(); + } +}; diff --git a/compiler/pipes/calc-bad-vars.cpp b/compiler/pipes/calc-bad-vars.cpp index b539b17df4..032dec6f69 100644 --- a/compiler/pipes/calc-bad-vars.cpp +++ b/compiler/pipes/calc-bad-vars.cpp @@ -531,6 +531,22 @@ class CalcBadVars { } } + static void calc_interruptible(const FuncCallGraph &call_graph) { + IdMap into_interruptible(call_graph.n); + IdMap to_parents(call_graph.n); + + for (const auto &func : call_graph.functions) { + if (func->is_interruptible) { + mark(call_graph.rev_graph, into_interruptible, func, to_parents); + } + } + + for (const auto &func : call_graph.functions) { + if (into_interruptible[func]) { + func->is_interruptible = true; + } + } + } static void calc_resumable(const FuncCallGraph &call_graph, const std::vector &dep_data) { for (int i = 0; i < call_graph.n; i++) { @@ -668,6 +684,7 @@ class CalcBadVars { { FuncCallGraph call_graph(std::move(functions), dep_datas); + calc_interruptible(call_graph); calc_resumable(call_graph, dep_datas); generate_bad_vars(call_graph, dep_datas); check_func_colors(call_graph); diff --git a/compiler/pipes/calc-func-dep.cpp b/compiler/pipes/calc-func-dep.cpp index ce7c76049c..b5104b3ecb 100644 --- a/compiler/pipes/calc-func-dep.cpp +++ b/compiler/pipes/calc-func-dep.cpp @@ -55,8 +55,9 @@ VertexPtr CalcFuncDepPass::on_enter_vertex(VertexPtr vertex) { // .dep is used to // 1) build call graph — we actually need only user-defined function // 2) build resumable call graph — to propagate resumable state and to calc resumable chains + // 3) build interruptible call graph - calc interruptible chains // adding all built-in calls won't affect codegen, it will just overhead call graphs and execution time - if (!other_function->is_extern() || other_function->is_resumable || other_function->is_imported_from_static_lib()) { + if (!other_function->is_extern() || other_function->is_resumable || other_function->is_imported_from_static_lib() || other_function->is_interruptible) { data.dep.push_back(other_function); } calls.push_back(other_function); diff --git a/compiler/pipes/code-gen.cpp b/compiler/pipes/code-gen.cpp index 500873ab66..e4db43bf9f 100644 --- a/compiler/pipes/code-gen.cpp +++ b/compiler/pipes/code-gen.cpp @@ -117,7 +117,10 @@ void CodeGenF::on_finish(DataStream> &os) { // TODO: should be done in lib mode also, but in some other way if (!G->is_output_mode_lib()) { - code_gen_start_root_task(os, std::make_unique(vk::singleton::get().flush_forkable_types(), vk::singleton::get().flush_waitable_types())); + if (!G->is_output_mode_k2_component()) { + code_gen_start_root_task(os, std::make_unique(vk::singleton::get().flush_forkable_types(), + vk::singleton::get().flush_waitable_types())); + } code_gen_start_root_task(os, std::make_unique(TypeHintShape::get_all_registered_keys())); code_gen_start_root_task(os, std::make_unique(std::move(all_json_encoders))); code_gen_start_root_task(os, std::make_unique()); @@ -129,9 +132,12 @@ void CodeGenF::on_finish(DataStream> &os) { code_gen_start_root_task(os, std::make_unique()); code_gen_start_root_task(os, std::make_unique()); - if (!G->is_output_mode_lib()) { + if (!G->is_output_mode_lib() && !G->is_output_mode_k2_component()) { code_gen_start_root_task(os, std::make_unique()); } + if (G->is_output_mode_k2_component()) { + code_gen_start_root_task(os, std::make_unique()); + } } void CodeGenF::prepare_generate_function(FunctionPtr func) { diff --git a/compiler/pipes/parse-and-apply-phpdoc.cpp b/compiler/pipes/parse-and-apply-phpdoc.cpp index 12c2eb2070..561dc194f6 100644 --- a/compiler/pipes/parse-and-apply-phpdoc.cpp +++ b/compiler/pipes/parse-and-apply-phpdoc.cpp @@ -281,6 +281,8 @@ class ParseAndApplyPhpDocForFunction { f_->cpp_variadic_call = true; } else if (token == "tl_common_h_dep") { f_->tl_common_h_dep = true; + } else if (token == "interruptible") { + f_->is_interruptible = true; } else { kphp_error(0, fmt_format("Unknown @kphp-extern-func-info {}", token)); } From 49a1bf48f766af04add35054d49e716e9e4f7a73 Mon Sep 17 00:00:00 2001 From: Dmitry Solomennikov <144122424+soloth@users.noreply.github.com> Date: Thu, 27 Jun 2024 16:18:32 +0300 Subject: [PATCH 40/89] Fix incorrect parse of integer values for double vars (#1024) Values like `{"a": 101}` were parsed incorrectly during JSON parsing. They're internally interpreted as being integer and during assignment to double variable it were bit-casted from `int` to `double`, which is wrong. This PR fixes this behavior. Now integer values are converted to double explicitly. --- runtime/from-json-processor.h | 2 +- tests/phpt/json/37_json_float_read.php | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 tests/phpt/json/37_json_float_read.php diff --git a/runtime/from-json-processor.h b/runtime/from-json-processor.h index 3c54975c3a..49dd106529 100644 --- a/runtime/from-json-processor.h +++ b/runtime/from-json-processor.h @@ -79,7 +79,7 @@ class FromJsonVisitor { on_input_type_mismatch(json); return; } - value = json.as_double(); + value = json.is_int() ? json.as_int() : json.as_double(); } void do_set(string &value, const mixed &json) noexcept { diff --git a/tests/phpt/json/37_json_float_read.php b/tests/phpt/json/37_json_float_read.php new file mode 100644 index 0000000000..5c77de2f7f --- /dev/null +++ b/tests/phpt/json/37_json_float_read.php @@ -0,0 +1,23 @@ +amount = $amount; + } + + public function getAmount(): float { + return $this->amount; + } +} + +function test_json_float_read(): void { + $raw = '{"amount":101}'; + + $res = JsonEncoder::decode($raw, MoneyRequest::class); + var_dump($res->getAmount()); +} + +test_json_float_read(); From 5597a004385dde623398f073d7f572bda6422b7f Mon Sep 17 00:00:00 2001 From: Vadim Sadokhov <65451602+astrophysik@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:48:02 +0300 Subject: [PATCH 41/89] fix runtime issues (#1034) --- CMakeLists.txt | 1 + runtime-core/core-types/decl/string_buffer_decl.inl | 5 +++-- runtime-core/runtime-core-context.h | 8 +++++++- runtime/memory_usage.h | 4 ++-- runtime/runtime.cmake | 5 +++++ 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c67c518da..6a44c5e272 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,6 +85,7 @@ endif () install(DIRECTORY ${COMMON_DIR} ${BASE_DIR}/runtime + ${BASE_DIR}/runtime-core ${BASE_DIR}/server ${BASE_DIR}/third-party COMPONENT KPHP diff --git a/runtime-core/core-types/decl/string_buffer_decl.inl b/runtime-core/core-types/decl/string_buffer_decl.inl index 17ad43919f..5a589162e2 100644 --- a/runtime-core/core-types/decl/string_buffer_decl.inl +++ b/runtime-core/core-types/decl/string_buffer_decl.inl @@ -63,8 +63,9 @@ public: friend inline bool operator!=(const string_buffer &lhs, const string_buffer &rhs); }; + struct string_buffer_lib_context { - string::size_type MIN_BUFFER_LEN; - string::size_type MAX_BUFFER_LEN; + string::size_type MIN_BUFFER_LEN = 266175; + string::size_type MAX_BUFFER_LEN = (1 << 24); int error_flag = 0; }; diff --git a/runtime-core/runtime-core-context.h b/runtime-core/runtime-core-context.h index a574e49280..a6ae90598f 100644 --- a/runtime-core/runtime-core-context.h +++ b/runtime-core/runtime-core-context.h @@ -34,7 +34,13 @@ struct RuntimeAllocator { }; struct KphpCoreContext { - + /** + * KphpCoreContext is used in + * @see init_php_scripts_once_in_master for runtime or + * @see vk_k2_create_image_state for runtime light + * + * before the init() function is called, so its default parameters should be as follows + **/ static KphpCoreContext& current() noexcept; void init(); diff --git a/runtime/memory_usage.h b/runtime/memory_usage.h index abfbf783ee..43888b1aae 100644 --- a/runtime/memory_usage.h +++ b/runtime/memory_usage.h @@ -176,7 +176,7 @@ inline int64_t f$memory_get_static_usage() { return static_cast(dl::get_heap_memory_used()); } -inline int64_t f$memory_get_peak_usage(bool real_usage) { +inline int64_t f$memory_get_peak_usage(bool real_usage = false) { if (real_usage) { return static_cast(dl::get_script_memory_stats().max_real_memory_used); } else { @@ -184,7 +184,7 @@ inline int64_t f$memory_get_peak_usage(bool real_usage) { } } -inline int64_t f$memory_get_usage(bool real_usage __attribute__((unused))) { +inline int64_t f$memory_get_usage(bool real_usage __attribute__((unused)) = false) { return static_cast(dl::get_script_memory_stats().memory_used); } diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 0ca8ebacf1..0739061987 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -174,6 +174,11 @@ file(GLOB_RECURSE KPHP_RUNTIME_ALL_HEADERS RELATIVE ${BASE_DIR} CONFIGURE_DEPENDS "${BASE_DIR}/runtime/*.h") +file(GLOB_RECURSE KPHP_RUNTIME_CORE_ALL_HEADERS + RELATIVE ${BASE_DIR} + CONFIGURE_DEPENDS + "${BASE_DIR}/runtime-core/*.h") +list(APPEND KPHP_RUNTIME_ALL_HEADERS ${KPHP_RUNTIME_CORE_ALL_HEADERS}) list(TRANSFORM KPHP_RUNTIME_ALL_HEADERS REPLACE "^(.+)$" [[#include "\1"]]) list(JOIN KPHP_RUNTIME_ALL_HEADERS "\n" MERGED_RUNTIME_HEADERS) file(WRITE ${AUTO_DIR}/runtime/runtime-headers.h "\ From 3348823027477f9c93efe142b0bbef1083668bc3 Mon Sep 17 00:00:00 2001 From: Denis Vaksman Date: Tue, 2 Jul 2024 19:03:30 +0300 Subject: [PATCH 42/89] check that global var type and its init value are the same (#1032) --- compiler/code-gen/files/global-vars-reset.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/compiler/code-gen/files/global-vars-reset.cpp b/compiler/code-gen/files/global-vars-reset.cpp index 1f1fdf4092..8c0bbbc392 100644 --- a/compiler/code-gen/files/global-vars-reset.cpp +++ b/compiler/code-gen/files/global-vars-reset.cpp @@ -10,6 +10,9 @@ #include "compiler/code-gen/namespace.h" #include "compiler/code-gen/vertex-compiler.h" #include "compiler/data/src-file.h" +#include "compiler/inferring/public.h" +#include "compiler/inferring/type-data.h" +#include "compiler/kphp_assert.h" GlobalVarsReset::GlobalVarsReset(const GlobalsBatchedMem &all_globals_in_mem) : all_globals_in_mem(all_globals_in_mem) {} @@ -41,6 +44,17 @@ void GlobalVarsReset::compile_globals_reset_part(CodeGenerator &W, const Globals W << "// " << var->as_human_readable() << NL; W << "hard_reset_var(" << GlobalVarInPhpGlobals(var); if (var->init_val) { + // TODO: fix unstable type inferring + const TypeData *global_var_type = tinf::get_type(var); + const TypeData *init_val_type = tinf::get_type(var->init_val); + // a -> b <=> !a || b + kphp_error(!(global_var_type->get_real_ptype() == tp_array && + init_val_type->get_real_ptype() == tp_array && + init_val_type->lookup_at_any_key()->get_real_ptype() != tp_any) + || are_equal_types(global_var_type, init_val_type), + fmt_format("Types of global variable and its init value differ: {} and {}.\n" + "Probably because of unstable type inferring, try rerun compilation", + global_var_type->as_human_readable(), init_val_type->as_human_readable())); W << ", " << var->init_val; } W << ");" << NL; From 2022e07dd51f77ce44f03ae2ba63f5d8094ad737 Mon Sep 17 00:00:00 2001 From: Alexander Polyakov Date: Fri, 5 Jul 2024 14:40:18 +0300 Subject: [PATCH 43/89] Runtime light: RPC support (#1030) Add preliminary support for RPC into runtime-light. Also, the PR fixes a bug in unsycnhronized_memory_resource. It prevents defragmentation from merging two memory pieces from distinct memory arenas, e.g. main memory resource and extra memory pool. --- builtin-functions/kphp-light/functions.txt | 78 ++- cmake/init-compilation-flags.cmake | 3 + compiler/code-gen/files/init-scripts.cpp | 1 - .../code-gen/files/tl2cpp/tl-combinator.cpp | 13 +- .../code-gen/files/tl2cpp/tl-combinator.h | 2 +- .../code-gen/files/tl2cpp/tl-function.cpp | 8 +- compiler/code-gen/files/tl2cpp/tl-type.cpp | 10 +- compiler/code-gen/files/tl2cpp/tl-type.h | 8 +- compiler/code-gen/files/tl2cpp/tl2cpp.cpp | 16 +- .../allocator/script-allocator-managed.h | 4 + .../details/memory_ordered_chunk_list.cpp | 11 +- .../details/memory_ordered_chunk_list.h | 3 +- .../memory-resource/extra-memory-pool.h | 6 +- .../monotonic_buffer_resource.h | 7 +- .../unsynchronized_pool_resource.cpp | 6 +- .../unsynchronized_pool_resource.h | 3 +- runtime-light/allocator/allocator.h | 14 + .../allocator/runtime-light-allocator.cpp | 28 +- runtime-light/component/component.h | 10 +- runtime-light/component/image.h | 6 +- runtime-light/coroutine/task.h | 2 +- runtime-light/runtime-light.cmake | 20 +- runtime-light/stdlib/rpc/rpc-api.cpp | 522 ++++++++++++++++ runtime-light/stdlib/rpc/rpc-api.h | 113 ++++ runtime-light/stdlib/rpc/rpc-context.cpp | 26 + runtime-light/stdlib/rpc/rpc-context.h | 67 +++ .../stdlib/rpc/rpc-extra-headers.cpp | 67 +++ runtime-light/stdlib/rpc/rpc-extra-headers.h | 38 ++ runtime-light/stdlib/rpc/rpc-extra-info.cpp | 29 + runtime-light/stdlib/rpc/rpc-extra-info.h | 30 + runtime-light/stdlib/rpc/rpc-tl-defs.h | 41 ++ runtime-light/stdlib/rpc/rpc-tl-error.cpp | 94 +++ runtime-light/stdlib/rpc/rpc-tl-error.h | 53 ++ runtime-light/stdlib/rpc/rpc-tl-func-base.h | 34 ++ runtime-light/stdlib/rpc/rpc-tl-function.h | 103 ++++ .../stdlib/rpc/rpc-tl-kphp-request.h | 60 ++ runtime-light/stdlib/rpc/rpc-tl-query.cpp | 85 +++ runtime-light/stdlib/rpc/rpc-tl-query.h | 36 ++ runtime-light/stdlib/rpc/rpc-tl-request.cpp | 47 ++ runtime-light/stdlib/rpc/rpc-tl-request.h | 56 ++ runtime-light/stdlib/stdlib.cmake | 7 + runtime-light/tl/tl-builtins.cpp | 204 +++++++ runtime-light/tl/tl-builtins.h | 560 ++++++++++++++++++ runtime-light/tl/tl.cmake | 3 + runtime-light/utils/concepts.h | 10 + runtime/rpc.cpp | 18 +- runtime/tl/rpc_request.h | 4 +- runtime/tl/rpc_tl_query.cpp | 10 +- runtime/tl/rpc_tl_query.h | 6 +- runtime/tl/tl_builtins.h | 48 +- runtime/typed_rpc.cpp | 6 +- .../details/memory_chunk_tree-test.cpp | 44 +- .../memory_ordered_chunk_list-test.cpp | 24 +- .../unsynchronized_pool_resource-test.cpp | 46 +- 54 files changed, 2597 insertions(+), 153 deletions(-) create mode 100644 runtime-light/allocator/allocator.h create mode 100644 runtime-light/stdlib/rpc/rpc-api.cpp create mode 100644 runtime-light/stdlib/rpc/rpc-api.h create mode 100644 runtime-light/stdlib/rpc/rpc-context.cpp create mode 100644 runtime-light/stdlib/rpc/rpc-context.h create mode 100644 runtime-light/stdlib/rpc/rpc-extra-headers.cpp create mode 100644 runtime-light/stdlib/rpc/rpc-extra-headers.h create mode 100644 runtime-light/stdlib/rpc/rpc-extra-info.cpp create mode 100644 runtime-light/stdlib/rpc/rpc-extra-info.h create mode 100644 runtime-light/stdlib/rpc/rpc-tl-defs.h create mode 100644 runtime-light/stdlib/rpc/rpc-tl-error.cpp create mode 100644 runtime-light/stdlib/rpc/rpc-tl-error.h create mode 100644 runtime-light/stdlib/rpc/rpc-tl-func-base.h create mode 100644 runtime-light/stdlib/rpc/rpc-tl-function.h create mode 100644 runtime-light/stdlib/rpc/rpc-tl-kphp-request.h create mode 100644 runtime-light/stdlib/rpc/rpc-tl-query.cpp create mode 100644 runtime-light/stdlib/rpc/rpc-tl-query.h create mode 100644 runtime-light/stdlib/rpc/rpc-tl-request.cpp create mode 100644 runtime-light/stdlib/rpc/rpc-tl-request.h create mode 100644 runtime-light/tl/tl-builtins.cpp create mode 100644 runtime-light/tl/tl-builtins.h create mode 100644 runtime-light/tl/tl.cmake create mode 100644 runtime-light/utils/concepts.h diff --git a/builtin-functions/kphp-light/functions.txt b/builtin-functions/kphp-light/functions.txt index 75289b9005..d92b9dad24 100644 --- a/builtin-functions/kphp-light/functions.txt +++ b/builtin-functions/kphp-light/functions.txt @@ -76,6 +76,53 @@ function get_hash_of_class (object $klass) ::: int; function strlen ($str ::: string) ::: int; +// === Rpc ======================================================================================== + +/** @kphp-tl-class */ +interface RpcFunction { + public function getTLFunctionName() : string; +} + +/** @kphp-tl-class */ +interface RpcFunctionReturnResult {} + +// type ReqResult <=> RpcResponse +/** @kphp-tl-class */ +interface RpcResponse { + public function getResult() : @tl\RpcFunctionReturnResult; + public function getHeader() : @tl\_common\Types\rpcResponseHeader; + public function getError() : @tl\_common\Types\rpcResponseError; + public function isError() : bool; +} + +/** + * 'KphpRpcRequestsExtraInfo' is a builtin KPHP class. It may accumulate extra information + * about RPC requests sent in both typed and untyped versions of rpc_tl_query builtins. + */ +final class KphpRpcRequestsExtraInfo { + /** + * 'get' returns an array of extra information (request size) about sent RPC requests. + * + * @return tuple(int)[] + */ + public function get (); +} + +/** @kphp-extern-func-info interruptible */ +function rpc_tl_query($actor ::: string, $arr ::: array, $timeout ::: float = -1.0, $ignore_answer ::: bool = false, \KphpRpcRequestsExtraInfo $requests_extra_info = null, $need_responses_extra_info ::: bool = false) ::: int[]; + +/** @kphp-extern-func-info interruptible */ +function typed_rpc_tl_query($actor ::: string, @tl\RpcFunction[] $query_functions, $timeout ::: float = -1.0, $ignore_answer ::: bool = false, \KphpRpcRequestsExtraInfo $requests_extra_info = null, $need_responses_extra_info ::: bool = false) ::: int[]; + +/** @kphp-extern-func-info interruptible */ +function rpc_tl_query_result($query_ids ::: array) ::: mixed[][]; + +/** @kphp-extern-func-info interruptible */ +function typed_rpc_tl_query_result(int[] $query_ids) ::: @tl\RpcResponse[]; + + +// === Component ================================================================================== + class ComponentQuery { private function __construct() ::: \ComponentQuery; } @@ -118,6 +165,34 @@ function component_stream_read_exact($stream ::: ComponentStream, $len ::: int) function component_close_stream($stream ::: ComponentStream) ::: void; function component_finish_stream_processing($stream ::: ComponentStream) ::: void; +// === Json ======================================================================================= + +class JsonEncoder { + const rename_policy = 'none'; + const visibility_policy = 'all'; + const skip_if_default = false; + const float_precision = 0; + + private function __construct(); + + public static function encode(object $instance, int $flags = 0, array $more = []) : string; + public static function decode(string $json, string $class_name) : instance<^2>; + public static function getLastError() : string; + + // JsonEncoderOrChild::encode(...) is actually replaced by JsonEncoder::to_json_impl('JsonEncoderOrChild', ...) + static function to_json_impl(string $encoder_tag, object $instance, int $flags = 0, array $more = []) ::: string; + + // JsonEncoderOrChild::decode(...) is actually replaced by JsonEncoder::from_json_impl('JsonEncoderOrChild', ...) + /** @kphp-extern-func-info cpp_template_call */ + static function from_json_impl(string $encoder_tag, string $json, string $class_name) ::: instance<^3>; +} + +function json_encode ($v ::: mixed, $options ::: int = 0) ::: string | false; + +function json_decode ($v ::: string, $assoc ::: bool = false) ::: mixed; + +// === Misc ======================================================================================= + /** @kphp-extern-func-info cpp_template_call */ function instance_cast(object $instance, $to_type ::: string) ::: instance<^2>; @@ -131,9 +206,6 @@ function warning($message ::: string) ::: void; /** @kphp-no-return */ function critical_error($message ::: string) ::: void; -function json_encode ($v ::: mixed, $options ::: int = 0) ::: string | false; -function json_decode ($v ::: string, $assoc ::: bool = false) ::: mixed; - function debug_print_string($str ::: string) ::: void; function byte_to_int($str ::: string) ::: ?int; diff --git a/cmake/init-compilation-flags.cmake b/cmake/init-compilation-flags.cmake index 35ae461a12..e019a35748 100644 --- a/cmake/init-compilation-flags.cmake +++ b/cmake/init-compilation-flags.cmake @@ -105,6 +105,9 @@ endif() add_compile_options(-Werror -Wall -Wextra -Wunused-function -Wfloat-conversion -Wno-sign-compare -Wuninitialized -Wno-redundant-move -Wno-missing-field-initializers) +if(COMPILE_RUNTIME_LIGHT) + add_compile_options(-Wno-vla-cxx-extension) +endif() if(NOT APPLE) check_cxx_compiler_flag(-gz=zlib DEBUG_COMPRESSION_IS_FOUND) diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index 8e6398c63d..1a7f1b0a33 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -238,7 +238,6 @@ void InitScriptsCpp::compile(CodeGenerator &W) const { } } - W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL; if (G->is_output_mode_k2_component()) { diff --git a/compiler/code-gen/files/tl2cpp/tl-combinator.cpp b/compiler/code-gen/files/tl2cpp/tl-combinator.cpp index 2320c3cd57..e003a91dbb 100644 --- a/compiler/code-gen/files/tl2cpp/tl-combinator.cpp +++ b/compiler/code-gen/files/tl2cpp/tl-combinator.cpp @@ -66,7 +66,7 @@ void CombinatorStore::gen_before_args_processing(CodeGenerator &W) const { W << "(void)tl_object;" << NL; if (combinator->is_function()) { W << fmt_format("f$store_int({:#010x});", static_cast(combinator->id)) << NL; - W << fmt_format("CurrentProcessingQuery::get().set_last_stored_tl_function_magic({:#010x});", static_cast(combinator->id)) << NL; + W << fmt_format("CurrentTlQuery::get().set_last_stored_tl_function_magic({:#010x});", static_cast(combinator->id)) << NL; } } @@ -84,7 +84,7 @@ void CombinatorStore::gen_arg_processing(CodeGenerator &W, const std::unique_ptr if (!value_check.empty()) { W << "if (" << value_check << ") " << BEGIN; W - << fmt_format(R"(CurrentProcessingQuery::get().raise_storing_error("Optional field %s of %s is not set, but corresponding fields mask bit is set", "{}", "{}");)", + << fmt_format(R"(CurrentTlQuery::get().raise_storing_error("Optional field %s of %s is not set, but corresponding fields mask bit is set", "{}", "{}");)", arg->name, combinator->name) << NL; W << "return" << (combinator->is_function() ? " {};" : ";") << NL; W << END << NL; @@ -100,21 +100,22 @@ void CombinatorStore::gen_arg_processing(CodeGenerator &W, const std::unique_ptr auto *as_type_var = arg->type_expr->as(); kphp_assert(as_type_var); if (!typed_mode) { + const auto *k2_tl_storers_prefix = G->is_output_mode_k2_component() ? "RpcImageState::get()." : ""; W << "auto _cur_arg = " << fmt_format("tl_arr_get(tl_object, {}, {}, {}L)", tl2cpp::register_tl_const_str(arg->name), arg->idx, tl2cpp::hash_tl_const_str(arg->name)) << ";" << NL; W << "string target_f_name = " << fmt_format("tl_arr_get(_cur_arg, {}, 0, {}L).as_string()", tl2cpp::register_tl_const_str("_"), tl2cpp::hash_tl_const_str("_")) << ";" << NL; - W << "if (!tl_storers_ht.has_key(target_f_name)) " << BEGIN - << "CurrentProcessingQuery::get().raise_storing_error(\"Function %s not found in tl-scheme\", target_f_name.c_str());" << NL + W << fmt_format("if (!{}tl_storers_ht.has_key(target_f_name)) ", k2_tl_storers_prefix) << BEGIN + << "CurrentTlQuery::get().raise_storing_error(\"Function %s not found in tl-scheme\", target_f_name.c_str());" << NL << "return {};" << NL << END << NL; - W << "const auto &storer_kv = tl_storers_ht.get_value(target_f_name);" << NL; + W << fmt_format("const auto &storer_kv = {}tl_storers_ht.get_value(target_f_name);", k2_tl_storers_prefix) << NL; W << "tl_func_state->" << combinator->get_var_num_arg(as_type_var->var_num)->name << ".fetcher = storer_kv(_cur_arg);" << NL; } else { W << "if (tl_object->$" << arg->name << ".is_null()) " << BEGIN - << R"(CurrentProcessingQuery::get().raise_storing_error("Field \")" << arg->name << R"(\" not found in tl object");)" << NL + << R"(CurrentTlQuery::get().raise_storing_error("Field \")" << arg->name << R"(\" not found in tl object");)" << NL << "return {};" << NL << END << NL; W << "tl_func_state->" << combinator->get_var_num_arg(as_type_var->var_num)->name << ".fetcher = " diff --git a/compiler/code-gen/files/tl2cpp/tl-combinator.h b/compiler/code-gen/files/tl2cpp/tl-combinator.h index f98182a6e9..bd9f3454d7 100644 --- a/compiler/code-gen/files/tl2cpp/tl-combinator.h +++ b/compiler/code-gen/files/tl2cpp/tl-combinator.h @@ -59,7 +59,7 @@ struct CombinatorGen { auto _cur_arg = tl_arr_get(tl_object, tl_str$query, 4, 1563700686); string target_f_name = tl_arr_get(_cur_arg, tl_str$_, 0, -2147483553).as_string(); if (!tl_storers_ht.has_key(target_f_name)) { - CurrentProcessingQuery::get().raise_storing_error("Function %s not found in tl-scheme", target_f_name.c_str()); + CurrentTlQuery::get().raise_storing_error("Function %s not found in tl-scheme", target_f_name.c_str()); return {}; } const auto &storer_kv = tl_storers_ht.get_value(target_f_name); diff --git a/compiler/code-gen/files/tl2cpp/tl-function.cpp b/compiler/code-gen/files/tl2cpp/tl-function.cpp index f89aa4ace5..6e8bcd444a 100644 --- a/compiler/code-gen/files/tl2cpp/tl-function.cpp +++ b/compiler/code-gen/files/tl2cpp/tl-function.cpp @@ -69,18 +69,18 @@ void TlFunctionDef::compile(CodeGenerator &W) const { } if (f->is_kphp_rpc_server_function() && needs_typed_fetch_store) { FunctionSignatureGenerator(W) << "std::unique_ptr " << struct_name << "::rpc_server_typed_fetch(" << get_php_runtime_type(f) << " *tl_object) " << BEGIN; - W << "CurrentProcessingQuery::get().set_current_tl_function(" << register_tl_const_str(f->name) << ");" << NL; + W << "CurrentTlQuery::get().set_current_tl_function(" << register_tl_const_str(f->name) << ");" << NL; W << "auto tl_func_state = make_unique_on_script_memory<" << struct_name << ">();" << NL; W << CombinatorFetch(f, CombinatorPart::LEFT, true); - W << "CurrentProcessingQuery::get().reset();" << NL; + W << "CurrentTlQuery::get().reset();" << NL; W << "return std::move(tl_func_state);" << NL; W << END << NL << NL; FunctionSignatureGenerator(W) << "void " << struct_name << "::rpc_server_typed_store(const class_instance<" << G->settings().tl_classname_prefix.get() << "RpcFunctionReturnResult> &tl_object_) " << BEGIN; - W << "CurrentProcessingQuery::get().set_current_tl_function(" << register_tl_const_str(f->name) << ");" << NL; + W << "CurrentTlQuery::get().set_current_tl_function(" << register_tl_const_str(f->name) << ");" << NL; W << "auto tl_object = tl_object_.template cast_to<" << get_php_runtime_type(f, false) << "_result>().get();" << NL; W << CombinatorStore(f, CombinatorPart::RIGHT, true); - W << "CurrentProcessingQuery::get().reset();" << NL; + W << "CurrentTlQuery::get().reset();" << NL; W << END << NL << NL; } } diff --git a/compiler/code-gen/files/tl2cpp/tl-type.cpp b/compiler/code-gen/files/tl2cpp/tl-type.cpp index 026fb903fd..4ac6dfde75 100644 --- a/compiler/code-gen/files/tl2cpp/tl-type.cpp +++ b/compiler/code-gen/files/tl2cpp/tl-type.cpp @@ -20,7 +20,7 @@ void TypeStore::compile(CodeGenerator &W) const { if (typed_mode) { W << "if (tl_object.is_null()) " << BEGIN - << "CurrentProcessingQuery::get().raise_storing_error(\"Instance expected, but false given while storing tl type\");" << NL + << "CurrentTlQuery::get().raise_storing_error(\"Instance expected, but false given while storing tl type\");" << NL << "return;" << NL << END << NL; } @@ -48,7 +48,7 @@ void TypeStore::compile(CodeGenerator &W) const { W << cpp_tl_struct_name("c_", c->name, template_str) << "::" << store_call << NL << END; } W << (first ? "" : " else ") << BEGIN - << "CurrentProcessingQuery::get().raise_storing_error(\"Invalid constructor %s of type %s\", " + << "CurrentTlQuery::get().raise_storing_error(\"Invalid constructor %s of type %s\", " << "tl_object.get_class(), \"" << type->name << "\");" << NL << END << NL; @@ -68,7 +68,7 @@ void TypeStore::compile(CodeGenerator &W) const { W << cpp_tl_struct_name("c_", c->name, template_str) << "::" << store_call << NL << END; } W << (first ? "" : " else ") << BEGIN - << "CurrentProcessingQuery::get().raise_storing_error(\"Invalid constructor %s of type %s\", " + << "CurrentTlQuery::get().raise_storing_error(\"Invalid constructor %s of type %s\", " << "c_name.c_str(), \"" << type->name << "\");" << NL << END << NL; } @@ -112,7 +112,7 @@ void TypeFetch::compile(CodeGenerator &W) const { if (default_constructor != nullptr) { W << "int pos = tl_parse_save_pos();" << NL; } - W << "auto magic = static_cast(rpc_fetch_int());" << NL; + W << "auto magic = static_cast(f$fetch_int());" << NL; W << "switch(magic) " << BEGIN; for (const auto &c : type->constructors) { if (c.get() == default_constructor) { @@ -150,7 +150,7 @@ void TypeFetch::compile(CodeGenerator &W) const { W << "tl_object = result;" << NL; } } else { - W << "CurrentProcessingQuery::get().raise_fetching_error(\"Incorrect magic of type " << type->name << ": 0x%08x\", magic);" << NL; + W << "CurrentTlQuery::get().raise_fetching_error(\"Incorrect magic of type " << type->name << ": 0x%08x\", magic);" << NL; } W << END << NL; W << END << NL; diff --git a/compiler/code-gen/files/tl2cpp/tl-type.h b/compiler/code-gen/files/tl2cpp/tl-type.h index 813d970109..7197deaa4b 100644 --- a/compiler/code-gen/files/tl2cpp/tl-type.h +++ b/compiler/code-gen/files/tl2cpp/tl-type.h @@ -22,7 +22,7 @@ void t_Either::store(const mixed &tl_object) f$store_int(0xdf3ecb3b); c_right::store(tl_object, std::move(X), std::move(Y)); } else { - CurrentProcessingQuery::get().raise_storing_error("Invalid constructor %s of type %s", c_name.c_str(), "Either"); + CurrentTlQuery::get().raise_storing_error("Invalid constructor %s of type %s", c_name.c_str(), "Either"); } } * Typed TL: @@ -37,7 +37,7 @@ void t_Either::typed_store(const PhpType &tl const typename right__::type *conv_obj = tl_object.template cast_to::type>().get(); c_right::typed_store(conv_obj, std::move(X), std::move(Y)); } else { - CurrentProcessingQuery::get().raise_storing_error("Invalid constructor %s of type %s", tl_object.get_class(), "Either"); + CurrentTlQuery::get().raise_storing_error("Invalid constructor %s of type %s", tl_object.get_class(), "Either"); } } */ @@ -75,7 +75,7 @@ array t_Either::fetch() { break; } default: { - CurrentProcessingQuery::get().raise_fetching_error("Incorrect magic of type Either: 0x%08x", magic); + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Either: 0x%08x", magic); } } return result; @@ -101,7 +101,7 @@ void t_Either::typed_fetch_to(PhpType &tl_ob break; } default: { - CurrentProcessingQuery::get().raise_fetching_error("Incorrect magic of type Either: 0x%08x", magic); + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Either: 0x%08x", magic); } } } diff --git a/compiler/code-gen/files/tl2cpp/tl2cpp.cpp b/compiler/code-gen/files/tl2cpp/tl2cpp.cpp index 11212a9e30..4f3c7ab5e7 100644 --- a/compiler/code-gen/files/tl2cpp/tl2cpp.cpp +++ b/compiler/code-gen/files/tl2cpp/tl2cpp.cpp @@ -4,8 +4,9 @@ #include "compiler/code-gen/files/tl2cpp/tl2cpp.h" -#include "common/tlo-parsing/tlo-parsing.h" +#include +#include "common/tlo-parsing/tlo-parsing.h" #include "compiler/code-gen/files/tl2cpp/tl-module.h" #include "compiler/code-gen/files/tl2cpp/tl2cpp-utils.h" #include "compiler/code-gen/naming.h" @@ -89,7 +90,7 @@ void write_rpc_server_functions(CodeGenerator &W) { W << deps << NL; W << ExternInclude{G->settings().runtime_headers.get()} << NL; FunctionSignatureGenerator(W) << "class_instance f$rpc_server_fetch_request() " << BEGIN; - W << "auto function_magic = static_cast(rpc_fetch_int());" << NL; + W << "auto function_magic = static_cast(f$fetch_int());" << NL; W << "switch(function_magic) " << BEGIN; for (const auto &f : kphp_functions) { W << fmt_format("case {:#010x}: ", static_cast(f->id)) << BEGIN; @@ -147,13 +148,20 @@ void write_tl_query_handlers(CodeGenerator &W) { // a hash table that contains all TL functions; // it's passed to the runtime just like the fetch wrapper W << "array gen$tl_storers_ht;" << NL; + // count the number of TL storers + FunctionSignatureGenerator(W) << "void fill_tl_storers_ht() " << BEGIN; + std::stringstream ss{}; // TODO: use std::transform_reduce or something like that + int32_t tl_storers_nums = 0; for (const auto &module_name : modules_with_functions) { + tl_storers_nums += modules[module_name].target_functions.size(); for (const auto &f : modules[module_name].target_functions) { - W << "gen$tl_storers_ht.set_value(" << register_tl_const_str(f->name) << ", " << "&" << cpp_tl_struct_name("f_", f->name) << "::store, " - << hash_tl_const_str(f->name) << "L);" << NL; + ss << "gen$tl_storers_ht.set_value(" << register_tl_const_str(f->name) << ", " << "&" << cpp_tl_struct_name("f_", f->name) << "::store, " + << hash_tl_const_str(f->name) << "L);\n"; } } + W << fmt_format("gen$tl_storers_ht.reserve({}, false);", tl_storers_nums) << NL; + W << ss.str(); W << END << NL; W << CloseFile(); } diff --git a/runtime-core/allocator/script-allocator-managed.h b/runtime-core/allocator/script-allocator-managed.h index eae0cb6125..35231dd273 100644 --- a/runtime-core/allocator/script-allocator-managed.h +++ b/runtime-core/allocator/script-allocator-managed.h @@ -1,3 +1,7 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + #pragma once #include diff --git a/runtime-core/memory-resource/details/memory_ordered_chunk_list.cpp b/runtime-core/memory-resource/details/memory_ordered_chunk_list.cpp index 2458355120..954208acfa 100644 --- a/runtime-core/memory-resource/details/memory_ordered_chunk_list.cpp +++ b/runtime-core/memory-resource/details/memory_ordered_chunk_list.cpp @@ -5,13 +5,15 @@ #include "runtime-core/memory-resource/details/memory_ordered_chunk_list.h" #include +#include #include namespace memory_resource { namespace details { -memory_ordered_chunk_list::memory_ordered_chunk_list(char *memory_resource_begin) noexcept: - memory_resource_begin_(memory_resource_begin) { +memory_ordered_chunk_list::memory_ordered_chunk_list(char *memory_resource_begin, char *memory_resource_end) noexcept + : memory_resource_begin_(memory_resource_begin) + , memory_resource_end_(memory_resource_end) { static_assert(sizeof(list_node) == 8, "8 bytes expected"); } @@ -33,7 +35,12 @@ void memory_ordered_chunk_list::add_from_array(list_node **first, list_node **la return; } + last = std::partition(first, last, [this](const auto *mem) { + return reinterpret_cast(mem) >= reinterpret_cast(this->memory_resource_begin_) + && reinterpret_cast(mem) < reinterpret_cast(this->memory_resource_end_); + }); std::sort(first, last, std::greater<>{}); + if (!head_) { head_ = *first++; } else if (reinterpret_cast(head_) < reinterpret_cast(*first)) { diff --git a/runtime-core/memory-resource/details/memory_ordered_chunk_list.h b/runtime-core/memory-resource/details/memory_ordered_chunk_list.h index da75fe1980..0694a22488 100644 --- a/runtime-core/memory-resource/details/memory_ordered_chunk_list.h +++ b/runtime-core/memory-resource/details/memory_ordered_chunk_list.h @@ -34,7 +34,7 @@ class memory_ordered_chunk_list : vk::not_copyable { uint32_t chunk_size_{0}; }; - explicit memory_ordered_chunk_list(char *memory_resource_begin) noexcept; + explicit memory_ordered_chunk_list(char *memory_resource_begin, char *memory_resource_end) noexcept; list_node *get_next(const list_node *node) const noexcept { return node->has_next() ? reinterpret_cast(memory_resource_begin_ + node->next_chunk_offset_) : nullptr; @@ -56,6 +56,7 @@ class memory_ordered_chunk_list : vk::not_copyable { void add_from_array(list_node **first, list_node **last) noexcept; char *memory_resource_begin_{nullptr}; + char *memory_resource_end_{nullptr}; list_node *head_{nullptr}; size_t tmp_buffer_size_{0}; std::array tmp_buffer_; diff --git a/runtime-core/memory-resource/extra-memory-pool.h b/runtime-core/memory-resource/extra-memory-pool.h index b868291532..e9a6149b61 100644 --- a/runtime-core/memory-resource/extra-memory-pool.h +++ b/runtime-core/memory-resource/extra-memory-pool.h @@ -6,8 +6,8 @@ #include #include -#include #include +#include #include "common/mixin/not_copyable.h" @@ -24,8 +24,8 @@ class alignas(8) extra_memory_pool : vk::not_copyable { } bool is_memory_from_this_pool(const void *mem, size_t mem_size) noexcept { - return memory_begin() <= static_cast(mem) && - static_cast(mem) + mem_size <= memory_begin() + get_pool_payload_size(); + return reinterpret_cast(memory_begin()) <= reinterpret_cast(mem) + && reinterpret_cast(mem) + mem_size <= reinterpret_cast(memory_begin()) + get_pool_payload_size(); } static size_t get_pool_payload_size(size_t buffer_size) noexcept { diff --git a/runtime-core/memory-resource/monotonic_buffer_resource.h b/runtime-core/memory-resource/monotonic_buffer_resource.h index 7eb9cdbad3..a3e1c2a779 100644 --- a/runtime-core/memory-resource/monotonic_buffer_resource.h +++ b/runtime-core/memory-resource/monotonic_buffer_resource.h @@ -41,9 +41,8 @@ class monotonic_buffer : vk::not_copyable { stats_.real_memory_used = static_cast(memory_current_ - memory_begin_); } - bool check_memory_piece(void *mem, size_t size) const noexcept { - return memory_begin_ <= static_cast(mem) && - static_cast(mem) + size <= memory_current_; + bool check_memory_piece_was_used(void *mem, size_t size) const noexcept { + return memory_begin_ <= static_cast(mem) && static_cast(mem) + size <= memory_current_; } MemoryStats stats_; @@ -98,7 +97,7 @@ class monotonic_buffer_resource : protected monotonic_buffer { } bool put_memory_back(void *mem, size_t size) noexcept { - if (unlikely(!check_memory_piece(mem, size))) { + if (unlikely(!check_memory_piece_was_used(mem, size))) { critical_dump(mem, size); } diff --git a/runtime-core/memory-resource/unsynchronized_pool_resource.cpp b/runtime-core/memory-resource/unsynchronized_pool_resource.cpp index 4c0a26481a..da5ba3b24e 100644 --- a/runtime-core/memory-resource/unsynchronized_pool_resource.cpp +++ b/runtime-core/memory-resource/unsynchronized_pool_resource.cpp @@ -36,10 +36,10 @@ void unsynchronized_pool_resource::unfreeze_oom_handling_memory() noexcept { void unsynchronized_pool_resource::perform_defragmentation() noexcept { memory_debug("perform memory defragmentation\n"); - details::memory_ordered_chunk_list mem_list{memory_begin_}; + details::memory_ordered_chunk_list mem_list{memory_begin_, memory_end_}; huge_pieces_.flush_to(mem_list); - if (const size_t fallback_resource_left_size = fallback_resource_.size()) { + if (const auto fallback_resource_left_size = fallback_resource_.size(); fallback_resource_left_size > 0) { mem_list.add_memory(fallback_resource_.memory_current(), fallback_resource_left_size); fallback_resource_.init(nullptr, 0); } @@ -75,7 +75,7 @@ void *unsynchronized_pool_resource::allocate_small_piece_from_fallback_resource( details::memory_chunk_tree::tree_node *smallest_huge_piece = huge_pieces_.extract_smallest(); if (!smallest_huge_piece) { perform_defragmentation(); - if ((mem = try_allocate_small_piece(aligned_size))) { + if (mem = try_allocate_small_piece(aligned_size); mem != nullptr) { return mem; } smallest_huge_piece = huge_pieces_.extract_smallest(); diff --git a/runtime-core/memory-resource/unsynchronized_pool_resource.h b/runtime-core/memory-resource/unsynchronized_pool_resource.h index cc4fb2db3e..434dbca671 100644 --- a/runtime-core/memory-resource/unsynchronized_pool_resource.h +++ b/runtime-core/memory-resource/unsynchronized_pool_resource.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include "runtime-core/memory-resource/extra-memory-pool.h" #include "runtime-core/memory-resource/details/memory_chunk_list.h" @@ -140,7 +141,7 @@ class unsynchronized_pool_resource : private monotonic_buffer_resource { extra_memory_pool *extra_memory_head_{nullptr}; extra_memory_pool extra_memory_tail_{sizeof(extra_memory_pool)}; - static constexpr size_t MAX_CHUNK_BLOCK_SIZE_{16u * 1024u}; + static constexpr size_t MAX_CHUNK_BLOCK_SIZE_{static_cast(16U * 1024U)}; std::array free_chunks_; }; diff --git a/runtime-light/allocator/allocator.h b/runtime-light/allocator/allocator.h new file mode 100644 index 0000000000..ca9d7fa3cb --- /dev/null +++ b/runtime-light/allocator/allocator.h @@ -0,0 +1,14 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-core/allocator/script-allocator-managed.h" + +template T, typename... Args> +requires std::constructible_from auto make_unique_on_script_memory(Args &&...args) noexcept { + return std::make_unique(std::forward(args)...); +} diff --git a/runtime-light/allocator/runtime-light-allocator.cpp b/runtime-light/allocator/runtime-light-allocator.cpp index 1e82f85392..50ac7625c0 100644 --- a/runtime-light/allocator/runtime-light-allocator.cpp +++ b/runtime-light/allocator/runtime-light-allocator.cpp @@ -2,26 +2,28 @@ // Copyright (c) 2024 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt +#include +#include + #include "runtime-core/runtime-core.h" #include "runtime-light/component/component.h" #include "runtime-light/utils/panic.h" -static constexpr size_t MIN_REQ_EXTRA_MEM_SIZE = 16 * 1024u; - namespace { +// TODO: make it depend on max chunk size, e.g. MIN_EXTRA_MEM_SIZE = f(MAX_CHUNK_SIZE); +constexpr auto MIN_EXTRA_MEM_SIZE = static_cast(32U * 1024U); // extra mem size should be greater than max chunk block size + bool is_script_allocator_available() { return get_component_context() != nullptr; } void request_extra_memory(size_t requested_size) { - ComponentState &rt_ctx = *get_component_context(); - size_t extra_mem_size = std::max(MIN_REQ_EXTRA_MEM_SIZE, requested_size); // extra mem size should be greater than max chunk block size - void *extra_mem = get_platform_context()->allocator.alloc(extra_mem_size); - if (extra_mem == nullptr) { - php_error("script OOM"); - } - rt_ctx.runtime_allocator.memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); + const size_t extra_mem_size = std::max(MIN_EXTRA_MEM_SIZE, requested_size); + auto &rt_alloc = RuntimeAllocator::current(); + auto *extra_mem = rt_alloc.alloc_global_memory(extra_mem_size); + rt_alloc.memory_resource.add_extra_memory(new (extra_mem) memory_resource::extra_memory_pool{extra_mem_size}); } + } // namespace RuntimeAllocator::RuntimeAllocator(size_t script_mem_size, size_t oom_handling_mem_size) { @@ -61,7 +63,7 @@ void *RuntimeAllocator::alloc_script_memory(size_t size) noexcept { void *ptr = rt_ctx.runtime_allocator.memory_resource.allocate(size); if (ptr == nullptr) { - request_extra_memory(size * 2); + request_extra_memory(size); ptr = rt_ctx.runtime_allocator.memory_resource.allocate(size); php_assert(ptr != nullptr); } @@ -77,7 +79,7 @@ void *RuntimeAllocator::alloc0_script_memory(size_t size) noexcept { ComponentState &rt_ctx = *get_component_context(); void *ptr = rt_ctx.runtime_allocator.memory_resource.allocate0(size); if (ptr == nullptr) { - request_extra_memory(size * 2); + request_extra_memory(size); ptr = rt_ctx.runtime_allocator.memory_resource.allocate0(size); php_assert(ptr != nullptr); } @@ -104,7 +106,7 @@ void RuntimeAllocator::free_script_memory(void *mem, size_t size) noexcept { php_assert(size); ComponentState &rt_ctx = *get_component_context(); - return rt_ctx.runtime_allocator.memory_resource.deallocate(mem, size); + rt_ctx.runtime_allocator.memory_resource.deallocate(mem, size); } void *RuntimeAllocator::alloc_global_memory(size_t size) noexcept { @@ -120,7 +122,7 @@ void *RuntimeAllocator::alloc0_global_memory(size_t size) noexcept { if (unlikely(ptr == nullptr)) { critical_error_handler(); } - memset(ptr, 0, size); + std::memset(ptr, 0, size); return ptr; } diff --git a/runtime-light/component/component.h b/runtime-light/component/component.h index 638274ac98..7057f6dd5a 100644 --- a/runtime-light/component/component.h +++ b/runtime-light/component/component.h @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -16,6 +17,7 @@ #include "runtime-light/core/globals/php-script-globals.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/output-control.h" +#include "runtime-light/stdlib/rpc/rpc-context.h" #include "runtime-light/stdlib/superglobals.h" #include "runtime-light/streams/streams.h" #include "runtime-light/utils/context.h" @@ -25,7 +27,7 @@ struct ComponentState { using unordered_map = memory_resource::stl::unordered_map; template using deque = memory_resource::stl::deque; - static constexpr int INIT_RUNTIME_ALLOCATOR_SIZE = 16 * 1024u; + static constexpr auto INIT_RUNTIME_ALLOCATOR_SIZE = static_cast(512U * 1024U); // 512KB ComponentState() : runtime_allocator(INIT_RUNTIME_ALLOCATOR_SIZE, 0) @@ -33,11 +35,12 @@ struct ComponentState { , opened_streams(unordered_map::allocator_type{runtime_allocator.memory_resource}) , awaiting_coroutines(unordered_map>::allocator_type{runtime_allocator.memory_resource}) , timer_callbacks(unordered_map>::allocator_type{runtime_allocator.memory_resource}) - , incoming_pending_queries(deque::allocator_type{runtime_allocator.memory_resource}) {} + , incoming_pending_queries(deque::allocator_type{runtime_allocator.memory_resource}) + , rpc_component_context(runtime_allocator.memory_resource) {} ~ComponentState() = default; - inline bool not_finished() const noexcept { + bool not_finished() const noexcept { return poll_status != PollStatus::PollFinishedOk && poll_status != PollStatus::PollFinishedError; } @@ -67,6 +70,7 @@ struct ComponentState { deque incoming_pending_queries; KphpCoreContext kphp_core_context; + RpcComponentContext rpc_component_context; private: bool is_stream_timer(uint64_t stream_d); diff --git a/runtime-light/component/image.h b/runtime-light/component/image.h index ec75c8d9c1..40025b9c40 100644 --- a/runtime-light/component/image.h +++ b/runtime-light/component/image.h @@ -5,5 +5,9 @@ #pragma once #include "runtime-light/header.h" +#include "runtime-light/stdlib/rpc/rpc-context.h" -struct ImageState {}; +struct ImageState { + char *c_linear_mem; + RpcImageState rpc_image_state; +}; diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index 1a6db0bec0..cc4b990e0c 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -150,7 +150,7 @@ struct task_t : public task_base_t { if constexpr (!std::is_void{}) { T *t = std::launder(reinterpret_cast(get_handle().promise().bytes)); const vk::final_action final_action([t] { t->~T(); }); - return std::move(*t); + return *t; } } diff --git a/runtime-light/runtime-light.cmake b/runtime-light/runtime-light.cmake index f0dd8f3305..c8d6d1c831 100644 --- a/runtime-light/runtime-light.cmake +++ b/runtime-light/runtime-light.cmake @@ -2,6 +2,7 @@ include(${BASE_DIR}/runtime-light/allocator/allocator.cmake) include(${BASE_DIR}/runtime-light/core/core.cmake) include(${BASE_DIR}/runtime-light/stdlib/stdlib.cmake) include(${BASE_DIR}/runtime-light/streams/streams.cmake) +include(${BASE_DIR}/runtime-light/tl/tl.cmake) include(${BASE_DIR}/runtime-light/utils/utils.cmake) prepend(MONOTOINC_LIGHT_BUFFER_RESOURCE_SRC ${BASE_DIR}/runtime-light/memory-resource-impl/ @@ -11,15 +12,16 @@ prepend(RUNTIME_COMPONENT_SRC ${BASE_DIR}/runtime-light/ component/component.cpp) set(RUNTIME_LIGHT_SRC ${RUNTIME_CORE_SRC} - ${RUNTIME_STDLIB_SRC} - ${RUNTIME_ALLOCATOR_SRC} - ${RUNTIME_COROUTINE_SRC} - ${RUNTIME_COMPONENT_SRC} - ${RUNTIME_STREAMS_SRC} - ${RUNTIME_UTILS_SRC} - ${RUNTIME_LANGUAGE_SRC} - ${MONOTOINC_LIGHT_BUFFER_RESOURCE_SRC} - ${BASE_DIR}/runtime-light/runtime-light.cpp) + ${RUNTIME_STDLIB_SRC} + ${RUNTIME_ALLOCATOR_SRC} + ${RUNTIME_COROUTINE_SRC} + ${RUNTIME_COMPONENT_SRC} + ${RUNTIME_STREAMS_SRC} + ${RUNTIME_TL_SRC} + ${RUNTIME_UTILS_SRC} + ${RUNTIME_LANGUAGE_SRC} + ${MONOTOINC_LIGHT_BUFFER_RESOURCE_SRC} + ${BASE_DIR}/runtime-light/runtime-light.cpp) vk_add_library(runtime-light OBJECT ${RUNTIME_LIGHT_SRC}) set_property(TARGET runtime-light PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/runtime-light/stdlib/rpc/rpc-api.cpp b/runtime-light/stdlib/rpc/rpc-api.cpp new file mode 100644 index 0000000000..0e676225ee --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-api.cpp @@ -0,0 +1,522 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/rpc/rpc-api.h" + +#include +#include +#include +#include +#include +#include + +#include "common/algorithms/find.h" +#include "common/rpc-error-codes.h" +#include "common/tl/constants/common.h" +#include "runtime-light/allocator/allocator.h" +#include "runtime-light/stdlib/rpc/rpc-context.h" +#include "runtime-light/stdlib/rpc/rpc-extra-headers.h" +#include "runtime-light/streams/interface.h" +#include "runtime-light/utils/concepts.h" + +namespace rpc_impl_ { + +constexpr int32_t MAX_TIMEOUT_S = 86400; + +// TODO: change uint64_t to string::size_type after moving it from uint32_t to uint64_t +constexpr uint64_t SMALL_STRING_MAX_LEN = 253; +constexpr uint64_t MEDIUM_STRING_MAX_LEN = (static_cast(1) << 24) - 1; +[[maybe_unused]] constexpr uint64_t LARGE_STRING_MAX_LEN = (static_cast(1) << 56) - 1; + +constexpr uint8_t LARGE_STRING_MAGIC = 0xff; +constexpr uint8_t MEDIUM_STRING_MAGIC = 0xfe; + +mixed mixed_array_get_value(const mixed &arr, const string &str_key, int64_t num_key) noexcept { + if (!arr.is_array()) { + return {}; + } + + if (const auto &elem{arr.get_value(num_key)}; !elem.is_null()) { + return elem; + } + if (const auto &elem{arr.get_value(str_key)}; !elem.is_null()) { + return elem; + } + return {}; +} + +bool rpc_fetch_remaining_enough(size_t len) noexcept { + return RpcComponentContext::get().fetch_state.remaining() >= len; +} + +array make_fetch_error(string &&error_msg, int32_t error_code) { + array res; + res.set_value(string{"__error", 7}, std::move(error_msg)); + res.set_value(string{"__error_code", 12}, error_code); + return res; +} + +template +std::optional fetch_trivial() noexcept { + if (!rpc_fetch_remaining_enough(sizeof(T))) { + return {}; // TODO: error handling + } + + auto &rpc_ctx{RpcComponentContext::get()}; + const auto v{*reinterpret_cast(rpc_ctx.buffer.c_str() + rpc_ctx.fetch_state.pos())}; + rpc_ctx.fetch_state.adjust(sizeof(T)); + return v; +} + +array fetch_function_untyped(const class_instance &rpc_query) noexcept { + php_assert(!rpc_query.is_null()); + CurrentTlQuery::get().set_current_tl_function(rpc_query); + auto fetcher{rpc_query.get()->result_fetcher->extract_untyped_fetcher()}; + php_assert(fetcher); + + const auto res{RpcImageState::get().tl_fetch_wrapper(std::move(fetcher))}; + // TODO: exception handling + // TODO: EOF handling + return res; +} + +class_instance fetch_function_typed(const class_instance &rpc_query, const RpcErrorFactory &error_factory) noexcept { + php_assert(!rpc_query.is_null()); + CurrentTlQuery::get().set_current_tl_function(rpc_query); + // check if the response is error + if (const auto rpc_error{error_factory.fetch_error_if_possible()}; !rpc_error.is_null()) { + return rpc_error; + } + const auto res{rpc_query.get()->result_fetcher->fetch_typed_response()}; + // TODO: exception handling + // TODO: EOF handling + return res; +} + +template +bool store_trivial(T v) noexcept { + RpcComponentContext::get().buffer.append(reinterpret_cast(&v), sizeof(T)); + return true; +} + +class_instance store_function(const mixed &tl_object) noexcept { + auto &cur_query{CurrentTlQuery::get()}; + const auto &rpc_image_state{RpcImageState::get()}; + + const auto fun_name{mixed_array_get_value(tl_object, string{"_"}, 0).to_string()}; // TODO: constexpr ctor for string{"_"} + if (!rpc_image_state.tl_storers_ht.has_key(fun_name)) { + cur_query.raise_storing_error("Function \"%s\" not found in tl-scheme", fun_name.c_str()); + return {}; + } + + auto rpc_tl_query{make_instance()}; + rpc_tl_query.get()->tl_function_name = fun_name; + + cur_query.set_current_tl_function(fun_name); + const auto &untyped_storer = rpc_image_state.tl_storers_ht.get_value(fun_name); + rpc_tl_query.get()->result_fetcher = make_unique_on_script_memory(untyped_storer(tl_object)); + cur_query.reset(); + return rpc_tl_query; +} + +task_t rpc_send_impl(const string &actor, double timeout, bool ignore_answer) noexcept { + auto &rpc_ctx = RpcComponentContext::get(); + + if (timeout <= 0 || timeout > MAX_TIMEOUT_S) { // TODO: handle timeouts + // timeout = conn.get()->timeout_ms * 0.001; + } + + string request_buf{}; + size_t request_size{rpc_ctx.buffer.size()}; + + // 'request_buf' will look like this: + // [ RpcExtraHeaders (optional) ] [ payload ] + if (const auto [opt_new_extra_header, cur_extra_header_size]{regularize_extra_headers(rpc_ctx.buffer.c_str(), ignore_answer)}; opt_new_extra_header) { + const auto new_extra_header{opt_new_extra_header.value()}; + const auto new_extra_header_size{sizeof(std::decay_t)}; + request_size = request_size - cur_extra_header_size + new_extra_header_size; + + request_buf.append(reinterpret_cast(&new_extra_header), new_extra_header_size); + request_buf.append(rpc_ctx.buffer.c_str() + cur_extra_header_size, rpc_ctx.buffer.size() - cur_extra_header_size); + } else { + request_buf.append(rpc_ctx.buffer.c_str(), request_size); + } + + // get timestamp before co_await to also count the time we were waiting for runtime to resume this coroutine + const auto timestamp{std::chrono::duration{std::chrono::system_clock::now().time_since_epoch()}.count()}; + + auto comp_query{co_await f$component_client_send_query(actor, request_buf)}; + if (comp_query.is_null()) { + php_error("could not send rpc query to %s", actor.c_str()); + co_return RpcQueryInfo{.id = RPC_INVALID_QUERY_ID, .request_size = request_size, .timestamp = timestamp}; + } + + if (ignore_answer) { // TODO: wait for answer in a separate coroutine and keep returning RPC_IGNORED_ANSWER_QUERY_ID + co_return RpcQueryInfo{.id = RPC_IGNORED_ANSWER_QUERY_ID, .request_size = request_size, .timestamp = timestamp}; + } + const auto query_id{rpc_ctx.current_query_id++}; + rpc_ctx.pending_component_queries.emplace(query_id, std::move(comp_query)); + co_return RpcQueryInfo{.id = query_id, .request_size = request_size, .timestamp = timestamp}; +} + +task_t rpc_tl_query_one_impl(const string &actor, mixed tl_object, double timeout, bool collect_resp_extra_info, bool ignore_answer) noexcept { + auto &rpc_ctx{RpcComponentContext::get()}; + + if (!tl_object.is_array()) { + rpc_ctx.current_query.raise_storing_error("not an array passed to function rpc_tl_query"); + co_return RpcQueryInfo{}; + } + + rpc_ctx.buffer.clean(); + auto rpc_tl_query{store_function(tl_object)}; // TODO: exception handling + if (rpc_tl_query.is_null()) { + co_return RpcQueryInfo{}; + } + + const auto query_info{co_await rpc_send_impl(actor, timeout, ignore_answer)}; + if (!ignore_answer) { + rpc_ctx.pending_rpc_queries.emplace(query_info.id, std::move(rpc_tl_query)); + } + if (collect_resp_extra_info) { + rpc_ctx.rpc_responses_extra_info.emplace(query_info.id, + std::make_pair(rpc_response_extra_info_status_t::NOT_READY, rpc_response_extra_info_t{0, query_info.timestamp})); + } + + co_return query_info; +} + +task_t typed_rpc_tl_query_one_impl(const string &actor, const RpcRequest &rpc_request, double timeout, bool collect_responses_extra_info, + bool ignore_answer) noexcept { + auto &rpc_ctx{RpcComponentContext::get()}; + + if (rpc_request.empty()) { + rpc_ctx.current_query.raise_storing_error("query function is null"); + co_return RpcQueryInfo{}; + } + + rpc_ctx.buffer.clean(); + auto fetcher{rpc_request.store_request()}; + if (!static_cast(fetcher)) { + rpc_ctx.current_query.raise_storing_error("could not store rpc request"); + co_return RpcQueryInfo{}; + } + + const auto query_info{co_await rpc_send_impl(actor, timeout, ignore_answer)}; + if (!ignore_answer) { + auto rpc_tl_query{make_instance()}; + rpc_tl_query.get()->result_fetcher = std::move(fetcher); + rpc_tl_query.get()->tl_function_name = rpc_request.tl_function_name(); + + rpc_ctx.pending_rpc_queries.emplace(query_info.id, std::move(rpc_tl_query)); + } + if (collect_responses_extra_info) { + rpc_ctx.rpc_responses_extra_info.emplace(query_info.id, + std::make_pair(rpc_response_extra_info_status_t::NOT_READY, rpc_response_extra_info_t{0, query_info.timestamp})); + } + + co_return query_info; +} + +task_t> rpc_tl_query_result_one_impl(int64_t query_id) noexcept { + if (query_id < RPC_VALID_QUERY_ID_RANGE_START) { + co_return make_fetch_error(string{"wrong query_id"}, TL_ERROR_WRONG_QUERY_ID); + } + + auto &rpc_ctx{RpcComponentContext::get()}; + class_instance rpc_query{}; + class_instance component_query{}; + + { + const auto it_rpc_query{rpc_ctx.pending_rpc_queries.find(query_id)}; + const auto it_component_query{rpc_ctx.pending_component_queries.find(query_id)}; + + vk::final_action finalizer{[&rpc_ctx, it_rpc_query, it_component_query]() { + rpc_ctx.pending_rpc_queries.erase(it_rpc_query); + rpc_ctx.pending_component_queries.erase(it_component_query); + }}; + + if (it_rpc_query == rpc_ctx.pending_rpc_queries.end() || it_component_query == rpc_ctx.pending_component_queries.end()) { + co_return make_fetch_error(string{"unexpectedly could not find query in pending queries"}, TL_ERROR_INTERNAL); + } + rpc_query = std::move(it_rpc_query->second); + component_query = std::move(it_component_query->second); + } + + if (rpc_query.is_null()) { + co_return make_fetch_error(string{"can't use rpc_tl_query_result for non-TL query"}, TL_ERROR_INTERNAL); + } + if (!rpc_query.get()->result_fetcher || rpc_query.get()->result_fetcher->empty()) { + co_return make_fetch_error(string{"rpc query has empty result fetcher"}, TL_ERROR_INTERNAL); + } + if (rpc_query.get()->result_fetcher->is_typed) { + co_return make_fetch_error(string{"can't get untyped result from typed TL query. Use consistent API for that"}, TL_ERROR_INTERNAL); + } + + const auto data{co_await f$component_client_get_result(component_query)}; + + // TODO: subscribe to rpc response event? + // update rpc response extra info + if (const auto it_response_extra_info{rpc_ctx.rpc_responses_extra_info.find(query_id)}; it_response_extra_info != rpc_ctx.rpc_responses_extra_info.end()) { + const auto timestamp{std::chrono::duration{std::chrono::system_clock::now().time_since_epoch()}.count()}; + it_response_extra_info->second.second = {data.size(), timestamp - std::get<1>(it_response_extra_info->second.second)}; + it_response_extra_info->second.first = rpc_response_extra_info_status_t::READY; + } + + rpc_ctx.fetch_state.reset(0, data.size()); + rpc_ctx.buffer.clean(); + rpc_ctx.buffer.append(data.c_str(), data.size()); + + co_return fetch_function_untyped(rpc_query); +} + +task_t> typed_rpc_tl_query_result_one_impl(int64_t query_id, const RpcErrorFactory &error_factory) noexcept { + if (query_id < RPC_VALID_QUERY_ID_RANGE_START) { + co_return error_factory.make_error(string{"wrong query_id"}, TL_ERROR_WRONG_QUERY_ID); + } + + auto &rpc_ctx{RpcComponentContext::get()}; + class_instance rpc_query{}; + class_instance component_query{}; + + { + const auto it_rpc_query{rpc_ctx.pending_rpc_queries.find(query_id)}; + const auto it_component_query{rpc_ctx.pending_component_queries.find(query_id)}; + + vk::final_action finalizer{[&rpc_ctx, it_rpc_query, it_component_query]() { + rpc_ctx.pending_rpc_queries.erase(it_rpc_query); + rpc_ctx.pending_component_queries.erase(it_component_query); + }}; + + if (it_rpc_query == rpc_ctx.pending_rpc_queries.end() || it_component_query == rpc_ctx.pending_component_queries.end()) { + co_return error_factory.make_error(string{"unexpectedly could not find query in pending queries"}, TL_ERROR_INTERNAL); + } + rpc_query = std::move(it_rpc_query->second); + component_query = std::move(it_component_query->second); + } + + if (rpc_query.is_null()) { + co_return error_factory.make_error(string{"can't use rpc_tl_query_result for non-TL query"}, TL_ERROR_INTERNAL); + } + if (!rpc_query.get()->result_fetcher || rpc_query.get()->result_fetcher->empty()) { + co_return error_factory.make_error(string{"rpc query has empty result fetcher"}, TL_ERROR_INTERNAL); + } + if (!rpc_query.get()->result_fetcher->is_typed) { + co_return error_factory.make_error(string{"can't get typed result from untyped TL query. Use consistent API for that"}, TL_ERROR_INTERNAL); + } + + const auto data{co_await f$component_client_get_result(component_query)}; + + // TODO: subscribe to rpc response event? + // update rpc response extra info + if (const auto it_response_extra_info{rpc_ctx.rpc_responses_extra_info.find(query_id)}; it_response_extra_info != rpc_ctx.rpc_responses_extra_info.end()) { + const auto timestamp{std::chrono::duration{std::chrono::system_clock::now().time_since_epoch()}.count()}; + it_response_extra_info->second.second = {data.size(), timestamp - std::get<1>(it_response_extra_info->second.second)}; + it_response_extra_info->second.first = rpc_response_extra_info_status_t::READY; + } + + rpc_ctx.fetch_state.reset(0, data.size()); + rpc_ctx.buffer.clean(); + rpc_ctx.buffer.append(data.c_str(), data.size()); + + co_return fetch_function_typed(rpc_query, error_factory); +} + +} // namespace rpc_impl_ + +// === Rpc Store ================================================================================== + +bool f$store_int(int64_t v) noexcept { + if (unlikely(is_int32_overflow(v))) { + php_warning("Got int32 overflow on storing '%" PRIi64 "', the value will be casted to '%d'", v, static_cast(v)); + } + return rpc_impl_::store_trivial(static_cast(v)); +} + +bool f$store_long(int64_t v) noexcept { + return rpc_impl_::store_trivial(v); +} + +bool f$store_float(double v) noexcept { + return rpc_impl_::store_trivial(static_cast(v)); +} + +bool f$store_double(double v) noexcept { + return rpc_impl_::store_trivial(v); +} + +bool f$store_string(const string &v) noexcept { // TODO: support large strings + auto &buffer{RpcComponentContext::get().buffer}; + + string::size_type string_len{v.size()}; + string::size_type size_len{}; + if (string_len <= rpc_impl_::SMALL_STRING_MAX_LEN) { + buffer << static_cast(string_len); + size_len = 1; + } else if (string_len <= rpc_impl_::MEDIUM_STRING_MAX_LEN) { + buffer << static_cast(rpc_impl_::MEDIUM_STRING_MAGIC) << static_cast(string_len & 0xff) << static_cast((string_len >> 8) & 0xff) + << static_cast((string_len >> 16) & 0xff); + size_len = 4; + } else { + buffer << static_cast(rpc_impl_::LARGE_STRING_MAGIC) << static_cast(string_len & 0xff) << static_cast((string_len >> 8) & 0xff) + << static_cast((string_len >> 16) & 0xff) << static_cast((string_len >> 24) & 0xff); +// buffer << static_cast(rpc_impl_::LARGE_STRING_MAGIC) << static_cast(string_len & 0xff) << static_cast((string_len >> 8) & 0xff) +// << static_cast((string_len >> 16) & 0xff) << static_cast((string_len >> 24) & 0xff) +// << static_cast((string_len >> 32) & 0xff) << static_cast((string_len >> 40) & 0xff) +// << static_cast((string_len >> 48) & 0xff); + size_len = 8; + } + buffer.append(v.c_str(), static_cast(string_len)); + + const auto total_len{size_len + string_len}; + const auto total_len_with_padding{(total_len + 3) & ~static_cast(3)}; + const auto padding{total_len_with_padding - total_len}; + + std::array padding_array{'\0', '\0', '\0', '\0'}; + buffer.append(padding_array.data(), padding); + return true; +} + +// === Rpc Fetch ================================================================================== + +int64_t f$fetch_int() noexcept { + const auto opt{rpc_impl_::fetch_trivial()}; + return opt.value_or(0); +} + +int64_t f$fetch_long() noexcept { + const auto opt{rpc_impl_::fetch_trivial()}; + return opt.value_or(0); +} + +double f$fetch_double() noexcept { + const auto opt{rpc_impl_::fetch_trivial()}; + return opt.value_or(0.0); +} + +double f$fetch_float() noexcept { + const auto opt{rpc_impl_::fetch_trivial()}; + return static_cast(opt.value_or(0.0)); +} + +string f$fetch_string() noexcept { + uint8_t first_byte{}; + if (const auto opt_first_byte{rpc_impl_::fetch_trivial()}; opt_first_byte) { + first_byte = opt_first_byte.value(); + } else { + return {}; // TODO: error handling + } + + string::size_type string_len{}; + string::size_type size_len{}; + switch (first_byte) { + case rpc_impl_::LARGE_STRING_MAGIC: { // next 7 bytes are string's length // TODO: support large strings + // static_assert(sizeof(string::size_type) >= 8, "string's length doesn't fit platform size"); + if (!rpc_impl_::rpc_fetch_remaining_enough(7)) { + return {}; // TODO: error handling + } + const auto first{static_cast(rpc_impl_::fetch_trivial().value())}; + const auto second{static_cast(rpc_impl_::fetch_trivial().value()) << 8}; + const auto third{static_cast(rpc_impl_::fetch_trivial().value()) << 16}; + const auto fourth{static_cast(rpc_impl_::fetch_trivial().value()) << 24}; + const auto fifth{static_cast(rpc_impl_::fetch_trivial().value()) << 32}; + const auto sixth{static_cast(rpc_impl_::fetch_trivial().value()) << 40}; + const auto seventh{static_cast(rpc_impl_::fetch_trivial().value()) << 48}; + string_len = first | second | third | fourth | fifth | sixth | seventh; + if (string_len < (1 << 24)) { + php_warning("long string's length is less than 1 << 24"); + } + size_len = 8; + } + case rpc_impl_::MEDIUM_STRING_MAGIC: { // next 3 bytes are string's length + if (!rpc_impl_::rpc_fetch_remaining_enough(3)) { + return {}; // TODO: error handling + } + const auto first{static_cast(rpc_impl_::fetch_trivial().value())}; + const auto second{static_cast(rpc_impl_::fetch_trivial().value()) << 8}; + const auto third{static_cast(rpc_impl_::fetch_trivial().value()) << 16}; + string_len = first | second | third; + if (string_len <= 253) { + php_warning("long string's length is less than 254"); + } + size_len = 4; + } + default: + string_len = static_cast(first_byte); + size_len = 1; + } + + const auto total_len_with_padding{(size_len + string_len + 3) & ~static_cast(3)}; + if (!rpc_impl_::rpc_fetch_remaining_enough(total_len_with_padding - size_len)) { + return {}; // TODO: error handling + } + + auto &rpc_ctx{RpcComponentContext::get()}; + string res{rpc_ctx.buffer.c_str() + rpc_ctx.fetch_state.pos(), string_len}; + rpc_ctx.fetch_state.adjust(total_len_with_padding - size_len); + return res; +} + +// === Rpc Query ================================================================================== + +task_t> f$rpc_tl_query(string actor, array tl_objects, double timeout, bool ignore_answer, + class_instance requests_extra_info, bool need_responses_extra_info) noexcept { + if (ignore_answer && need_responses_extra_info) { + php_warning("Both $ignore_answer and $need_responses_extra_info are 'true'. Can't collect metrics for ignored answers"); + } + + bool collect_resp_extra_info = !ignore_answer && need_responses_extra_info; + array query_ids{tl_objects.size()}; + array req_extra_info_arr{tl_objects.size()}; + + for (const auto &it : tl_objects) { + const auto query_info{co_await rpc_impl_::rpc_tl_query_one_impl(actor, it.get_value(), timeout, collect_resp_extra_info, ignore_answer)}; + query_ids.set_value(it.get_key(), query_info.id); + req_extra_info_arr.set_value(it.get_key(), rpc_request_extra_info_t{query_info.request_size}); + } + + if (!requests_extra_info.is_null()) { + requests_extra_info->extra_info_arr = std::move(req_extra_info_arr); + } + co_return query_ids; +} + +task_t>> f$rpc_tl_query_result(array query_ids) noexcept { + array> res{query_ids.size()}; + for (const auto &it : query_ids) { + res.set_value(it.get_key(), co_await rpc_impl_::rpc_tl_query_result_one_impl(it.get_value())); + } + co_return res; +} + +// === Rpc Misc ================================================================================== + +void f$rpc_clean() noexcept { + auto &rpc_ctx{RpcComponentContext::get()}; + rpc_ctx.buffer.clean(); + rpc_ctx.fetch_state.reset(0, 0); +} + +// === Misc ======================================================================================= + +bool is_int32_overflow(int64_t v) noexcept { + // f$store_int function is used for int and 'magic' storing, + // 'magic' can be assigned via hex literals which may set the 32nd bit, + // this is why we additionally check for the uint32_t here + const auto v32 = static_cast(v); + return vk::none_of_equal(v, int64_t{v32}, int64_t{static_cast(v32)}); +} + +void store_raw_vector_double(const array &vector) noexcept { // TODO: didn't we forget vector's length? + RpcComponentContext::get().buffer.append(reinterpret_cast(vector.get_const_vector_pointer()), sizeof(double) * vector.count()); +} + +void fetch_raw_vector_double(array &vector, int64_t num_elems) noexcept { + const auto len_bytes{sizeof(double) * num_elems}; + if (!rpc_impl_::rpc_fetch_remaining_enough(len_bytes)) { + return; // TODO: error handling + } + auto &rpc_ctx{RpcComponentContext::get()}; + vector.memcpy_vector(num_elems, rpc_ctx.buffer.c_str() + rpc_ctx.fetch_state.pos()); + rpc_ctx.fetch_state.adjust(len_bytes); +} diff --git a/runtime-light/stdlib/rpc/rpc-api.h b/runtime-light/stdlib/rpc/rpc-api.h new file mode 100644 index 0000000000..2bf19203af --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-api.h @@ -0,0 +1,113 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include + +#include "runtime-core/runtime-core.h" +#include "runtime-light/coroutine/task.h" +#include "runtime-light/stdlib/rpc/rpc-extra-info.h" +#include "runtime-light/stdlib/rpc/rpc-tl-error.h" +#include "runtime-light/stdlib/rpc/rpc-tl-function.h" +#include "runtime-light/stdlib/rpc/rpc-tl-kphp-request.h" + +constexpr int64_t RPC_VALID_QUERY_ID_RANGE_START = 0; +constexpr int64_t RPC_INVALID_QUERY_ID = -1; +constexpr int64_t RPC_IGNORED_ANSWER_QUERY_ID = -2; + +namespace rpc_impl_ { + +struct RpcQueryInfo { + int64_t id{RPC_INVALID_QUERY_ID}; + size_t request_size{0}; + double timestamp{0.0}; +}; + +task_t typed_rpc_tl_query_one_impl(const string &actor, const RpcRequest &rpc_request, double timeout, bool collect_responses_extra_info, + bool ignore_answer) noexcept; + +task_t> typed_rpc_tl_query_result_one_impl(int64_t query_id, const RpcErrorFactory &error_factory) noexcept; + +} // namespace rpc_impl_ + +// === Rpc Store ================================================================================== + +bool f$store_int(int64_t v) noexcept; + +bool f$store_long(int64_t v) noexcept; + +bool f$store_float(double v) noexcept; + +bool f$store_double(double v) noexcept; + +bool f$store_string(const string &v) noexcept; + +// === Rpc Fetch ================================================================================== + +int64_t f$fetch_int() noexcept; + +int64_t f$fetch_long() noexcept; + +double f$fetch_double() noexcept; + +double f$fetch_float() noexcept; + +string f$fetch_string() noexcept; + +// === Rpc Query ================================================================================== + +task_t> f$rpc_tl_query(string actor, array tl_objects, double timeout = -1.0, bool ignore_answer = false, + class_instance requests_extra_info = {}, bool need_responses_extra_info = false) noexcept; + +template rpc_function_t, std::same_as rpc_request_t = KphpRpcRequest> +task_t> f$typed_rpc_tl_query(string actor, array> query_functions, double timeout = -1.0, + bool ignore_answer = false, class_instance requests_extra_info = {}, + bool need_responses_extra_info = false) noexcept { + if (ignore_answer && need_responses_extra_info) { + php_warning("Both $ignore_answer and $need_responses_extra_info are 'true'. Can't collect metrics for ignored answers"); + } + + bool collect_resp_extra_info = !ignore_answer && need_responses_extra_info; + array query_ids{query_functions.size()}; + array req_extra_info_arr{query_functions.size()}; + + for (const auto &it : query_functions) { + const auto query_info{ + co_await rpc_impl_::typed_rpc_tl_query_one_impl(actor, rpc_request_t{it.get_value()}, timeout, collect_resp_extra_info, ignore_answer)}; + query_ids.set_value(it.get_key(), query_info.id); + req_extra_info_arr.set_value(it.get_key(), rpc_request_extra_info_t{query_info.request_size}); + } + + if (!requests_extra_info.is_null()) { + requests_extra_info->extra_info_arr = std::move(req_extra_info_arr); + } + co_return query_ids; +} + +task_t>> f$rpc_tl_query_result(array query_ids) noexcept; + +template query_id_t = int64_t, std::same_as error_factory_t = RpcResponseErrorFactory> +requires std::default_initializable task_t>> +f$typed_rpc_tl_query_result(array query_ids) noexcept { + array> res{query_ids.size()}; + for (const auto &it : query_ids) { + res.set_value(it.get_key(), co_await rpc_impl_::typed_rpc_tl_query_result_one_impl(it.get_value(), error_factory_t{})); + } + co_return res; +} + +// === Rpc Misc =================================================================================== + +void f$rpc_clean() noexcept; + +// === Misc ======================================================================================= + +bool is_int32_overflow(int64_t v) noexcept; + +void store_raw_vector_double(const array &vector) noexcept; + +void fetch_raw_vector_double(array &vector, int64_t num_elems) noexcept; diff --git a/runtime-light/stdlib/rpc/rpc-context.cpp b/runtime-light/stdlib/rpc/rpc-context.cpp new file mode 100644 index 0000000000..6f3c667329 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-context.cpp @@ -0,0 +1,26 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/rpc/rpc-context.h" + +#include "runtime-light/component/component.h" +#include "runtime-light/component/image.h" + +RpcComponentContext::RpcComponentContext(memory_resource::unsynchronized_pool_resource &memory_resource) + : current_query() + , pending_component_queries(unordered_map>::allocator_type{memory_resource}) + , pending_rpc_queries(unordered_map>::allocator_type{memory_resource}) + , rpc_responses_extra_info(unordered_map>::allocator_type{memory_resource}) {} + +RpcComponentContext &RpcComponentContext::get() noexcept { + return get_component_context()->rpc_component_context; +} + +const RpcImageState &RpcImageState::get() noexcept { + return get_image_state()->rpc_image_state; +} + +RpcImageState &RpcImageState::get_mutable() noexcept { + return get_mutable_image_state()->rpc_image_state; +} diff --git a/runtime-light/stdlib/rpc/rpc-context.h b/runtime-light/stdlib/rpc/rpc-context.h new file mode 100644 index 0000000000..f802246854 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-context.h @@ -0,0 +1,67 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "common/mixin/not_copyable.h" +#include "runtime-core/memory-resource/resource_allocator.h" +#include "runtime-core/runtime-core.h" +#include "runtime-light/stdlib/rpc/rpc-extra-info.h" +#include "runtime-light/stdlib/rpc/rpc-tl-defs.h" +#include "runtime-light/stdlib/rpc/rpc-tl-query.h" +#include "runtime-light/streams/component-stream.h" + +struct RpcComponentContext final : private vk::not_copyable { + class FetchState { + size_t m_pos{0}; + size_t m_remaining{0}; + + public: + constexpr FetchState() = default; + + constexpr size_t remaining() const noexcept { + return m_remaining; + } + + constexpr size_t pos() const noexcept { + return m_pos; + } + + constexpr void reset(size_t pos, size_t len) noexcept { + m_pos = pos; + m_remaining = len; + } + + constexpr void adjust(size_t len) noexcept { + m_pos += len; + m_remaining -= len; + } + }; + + template + using unordered_map = memory_resource::stl::unordered_map; + + string_buffer buffer; + FetchState fetch_state; + int64_t current_query_id{0}; + CurrentTlQuery current_query; + unordered_map> pending_component_queries; + unordered_map> pending_rpc_queries; + unordered_map> rpc_responses_extra_info; + + explicit RpcComponentContext(memory_resource::unsynchronized_pool_resource &memory_resource); + + static RpcComponentContext &get() noexcept; +}; + +struct RpcImageState final : private vk::not_copyable { + array tl_storers_ht; + tl_fetch_wrapper_ptr tl_fetch_wrapper{nullptr}; + + static const RpcImageState &get() noexcept; + static RpcImageState &get_mutable() noexcept; +}; diff --git a/runtime-light/stdlib/rpc/rpc-extra-headers.cpp b/runtime-light/stdlib/rpc/rpc-extra-headers.cpp new file mode 100644 index 0000000000..60cf80e946 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-extra-headers.cpp @@ -0,0 +1,67 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/rpc/rpc-extra-headers.h" + +#include + +#include "common/algorithms/find.h" +#include "common/tl/constants/common.h" +#include "runtime-light/utils/php_assert.h" + +namespace { + +constexpr int64_t EXPECTED_ACTOR_ID = 0; +constexpr uint32_t EMPTY_FLAGS = 0x0; + +} // namespace + +std::pair, uint32_t> regularize_extra_headers(const char *rpc_payload, bool ignore_result) noexcept { + const auto magic{*reinterpret_cast(rpc_payload)}; + if (vk::none_of_equal(magic, TL_RPC_DEST_ACTOR, TL_RPC_DEST_FLAGS, TL_RPC_DEST_ACTOR_FLAGS)) { + return {std::nullopt, 0}; + } + + uint32_t cur_extra_header_size{0}; + uint32_t cur_extra_header_flags{EMPTY_FLAGS}; + int64_t cur_extra_header_actor_id{EXPECTED_ACTOR_ID}; + switch (magic) { + case TL_RPC_DEST_ACTOR_FLAGS: { + cur_extra_header_size = sizeof(RpcDestActorFlagsHeaders); + const auto cur_wrapper{*reinterpret_cast(rpc_payload)}; + cur_extra_header_flags = cur_wrapper.flags; + cur_extra_header_actor_id = cur_wrapper.actor_id; + break; + } + case TL_RPC_DEST_ACTOR: { + cur_extra_header_size = sizeof(RpcDestActorHeaders); + const auto cur_wrapper{*reinterpret_cast(rpc_payload)}; + cur_extra_header_actor_id = cur_wrapper.actor_id; + break; + } + case TL_RPC_DEST_FLAGS: { + cur_extra_header_size = sizeof(RpcDestFlagsHeaders); + const auto cur_wrapper{*reinterpret_cast(rpc_payload)}; + cur_extra_header_flags = cur_wrapper.flags; + break; + } + default: { + php_critical_error("unreachable path"); + } + } + + if (cur_extra_header_actor_id != EXPECTED_ACTOR_ID) { + php_warning("RPC extra headers have actor_id set to %" PRId64 ", but it should not be explicitly set", cur_extra_header_actor_id); + } + const auto cur_extra_header_ignore_result{static_cast(cur_extra_header_flags & vk::tl::common::rpc_invoke_req_extra_flags::no_result)}; + if (!ignore_result && cur_extra_header_ignore_result) { + php_warning("inaccurate use of 'ignore_answer': 'false' was passed into TL query function (e.g., rpc_tl_query), " + "but 'true' was already set in RpcDestFlags or RpcDestActorFlags\n"); + } + + return {RpcDestActorFlagsHeaders{.op = TL_RPC_DEST_ACTOR_FLAGS, + .actor_id = EXPECTED_ACTOR_ID, + .flags = cur_extra_header_flags & ~vk::tl::common::rpc_invoke_req_extra_flags::no_result}, + cur_extra_header_size}; +} diff --git a/runtime-light/stdlib/rpc/rpc-extra-headers.h b/runtime-light/stdlib/rpc/rpc-extra-headers.h new file mode 100644 index 0000000000..dada642143 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-extra-headers.h @@ -0,0 +1,38 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include + +#pragma pack(push, 1) + +struct RpcDestActorFlagsHeaders { + uint32_t op; + int64_t actor_id; + uint32_t flags; +}; + +struct RpcDestActorHeaders { + uint32_t op; + int64_t actor_id; +}; + +struct RpcDestFlagsHeaders { + uint32_t op; + uint32_t flags; +}; + +#pragma pack(pop) + +/** + * Check RPC payload whether it contains some extra header. If so: + * 1) check if actor_id is set in the header; warn if it's set and not equal to 0; + * 2) check if ignore_result is set in the header; warn if it's set in the header and not set in [typed_]rpc_tl_query call; + * 3) return \ pair. + * Otherwise, return \. + * */ +std::pair, uint32_t> regularize_extra_headers(const char *rpc_payload, bool ignore_result) noexcept; diff --git a/runtime-light/stdlib/rpc/rpc-extra-info.cpp b/runtime-light/stdlib/rpc/rpc-extra-info.cpp new file mode 100644 index 0000000000..f30401c16f --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-extra-info.cpp @@ -0,0 +1,29 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/rpc/rpc-extra-info.h" + +#include "runtime-light/stdlib/rpc/rpc-context.h" + +const char *C$KphpRpcRequestsExtraInfo::get_class() const noexcept { + return R"(KphpRpcRequestsExtraInfo)"; +} + +int C$KphpRpcRequestsExtraInfo::get_hash() const noexcept { + return static_cast(vk::std_hash(vk::string_view(C$KphpRpcRequestsExtraInfo::get_class()))); +} + +array f$KphpRpcRequestsExtraInfo$$get(class_instance v$this) noexcept { + return v$this.get()->extra_info_arr; +} + +Optional f$extract_kphp_rpc_response_extra_info(int64_t query_id) noexcept { + auto &extra_info_map{RpcComponentContext::get().rpc_responses_extra_info}; + if (const auto it{extra_info_map.find(query_id)}; it != extra_info_map.end() && it->second.first == rpc_response_extra_info_status_t::READY) { + const auto extra_info{it->second.second}; + extra_info_map.erase(it); + return extra_info; + } + return {}; +} diff --git a/runtime-light/stdlib/rpc/rpc-extra-info.h b/runtime-light/stdlib/rpc/rpc-extra-info.h new file mode 100644 index 0000000000..7f91818f26 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-extra-info.h @@ -0,0 +1,30 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "common/algorithms/hashes.h" +#include "common/wrappers/string_view.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" + +using rpc_request_extra_info_t = std::tuple; // tuple(request_size) +using rpc_response_extra_info_t = std::tuple; // tuple(response_size, response_time) +enum class rpc_response_extra_info_status_t : uint8_t { NOT_READY, READY }; + +// TODO: visitors +struct C$KphpRpcRequestsExtraInfo final : public refcountable_php_classes /*, private DummyVisitorMethods */ { + array extra_info_arr; + + C$KphpRpcRequestsExtraInfo() = default; + const char *get_class() const noexcept; + int get_hash() const noexcept; +}; + +array f$KphpRpcRequestsExtraInfo$$get(class_instance v$this) noexcept; + +Optional f$extract_kphp_rpc_response_extra_info(int64_t query_id) noexcept; diff --git a/runtime-light/stdlib/rpc/rpc-tl-defs.h b/runtime-light/stdlib/rpc/rpc-tl-defs.h new file mode 100644 index 0000000000..9ad4258f93 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-defs.h @@ -0,0 +1,41 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-core/runtime-core.h" +#include "runtime-light/stdlib/rpc/rpc-tl-func-base.h" +#include "runtime-light/stdlib/rpc/rpc-tl-function.h" +#include "runtime-light/utils/php_assert.h" + +using tl_undefined_php_type = std::nullptr_t; +using tl_storer_ptr = std::unique_ptr (*)(const mixed &); +using tl_fetch_wrapper_ptr = array (*)(std::unique_ptr); + +struct tl_exclamation_fetch_wrapper { + std::unique_ptr fetcher; + + explicit tl_exclamation_fetch_wrapper(std::unique_ptr fetcher) + : fetcher(std::move(fetcher)) {} + + tl_exclamation_fetch_wrapper() noexcept = default; + tl_exclamation_fetch_wrapper(const tl_exclamation_fetch_wrapper &) = delete; + tl_exclamation_fetch_wrapper(tl_exclamation_fetch_wrapper &&) noexcept = default; + tl_exclamation_fetch_wrapper &operator=(const tl_exclamation_fetch_wrapper &) = delete; + tl_exclamation_fetch_wrapper &operator=(tl_exclamation_fetch_wrapper &&) noexcept = delete; + ~tl_exclamation_fetch_wrapper() = default; + + mixed fetch() const { + return fetcher->fetch(); + } + + using PhpType = class_instance; + + void typed_fetch_to(PhpType &out) const { + php_assert(fetcher); + out = fetcher->typed_fetch(); + } +}; diff --git a/runtime-light/stdlib/rpc/rpc-tl-error.cpp b/runtime-light/stdlib/rpc/rpc-tl-error.cpp new file mode 100644 index 0000000000..b1299f76b6 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-error.cpp @@ -0,0 +1,94 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/rpc/rpc-tl-error.h" + +#include + +#include "common/tl/constants/common.h" +#include "runtime-light/stdlib/rpc/rpc-api.h" +#include "runtime-light/tl/tl-builtins.h" + +bool TlRpcError::try_fetch() noexcept { + const auto backup_pos{tl_parse_save_pos()}; + auto op{f$fetch_int()}; + if (op == TL_REQ_RESULT_HEADER) { + fetch_and_skip_header(); + op = f$fetch_int(); + } + if (op != TL_RPC_REQ_ERROR) { + tl_parse_restore_pos(backup_pos); + return false; + } + + std::ignore = f$fetch_long(); + error_code = static_cast(f$fetch_int()); + error_msg = f$fetch_string(); + + // TODO: exception handling + return true; +} + +void TlRpcError::fetch_and_skip_header() const noexcept { + const auto flags{static_cast(f$fetch_int())}; + + if (flags & vk::tl::common::rpc_req_result_extra_flags::binlog_pos) { + std::ignore = f$fetch_long(); + } + if (flags & vk::tl::common::rpc_req_result_extra_flags::binlog_time) { + std::ignore = f$fetch_long(); + } + if (flags & vk::tl::common::rpc_req_result_extra_flags::engine_pid) { + std::ignore = f$fetch_int(); + std::ignore = f$fetch_int(); + std::ignore = f$fetch_int(); + } + if (flags & vk::tl::common::rpc_req_result_extra_flags::request_size) { + std::ignore = f$fetch_int(); + } + if (flags & vk::tl::common::rpc_req_result_extra_flags::response_size) { + std::ignore = f$fetch_int(); + } + if (flags & vk::tl::common::rpc_req_result_extra_flags::failed_subqueries) { + std::ignore = f$fetch_int(); + } + if (flags & vk::tl::common::rpc_req_result_extra_flags::compression_version) { + std::ignore = f$fetch_int(); + } + if (flags & vk::tl::common::rpc_req_result_extra_flags::stats) { + const auto size{f$fetch_int()}; + for (auto i = 0; i < size; ++i) { + std::ignore = f$fetch_int(); + std::ignore = f$fetch_int(); + } + } + if (flags & vk::tl::common::rpc_req_result_extra_flags::epoch_number) { + std::ignore = f$fetch_long(); + } + if (flags & vk::tl::common::rpc_req_result_extra_flags::view_number) { + std::ignore = f$fetch_long(); + } +} + +class_instance RpcErrorFactory::make_error(const char *error, int32_t error_code) const noexcept { + return make_error(string{error}, error_code); +} + +class_instance RpcErrorFactory::make_error_from_exception_if_possible() const noexcept { + // TODO + // if (!CurException.is_null()) { + // auto rpc_error = make_error(CurException->$message, TL_ERROR_SYNTAX); + // CurException = Optional{}; + // return rpc_error; + // } + return {}; +} + +class_instance RpcErrorFactory::fetch_error_if_possible() const noexcept { + TlRpcError rpc_error{}; + if (!rpc_error.try_fetch()) { + return {}; + } + return make_error(rpc_error.error_msg, rpc_error.error_code); +} diff --git a/runtime-light/stdlib/rpc/rpc-tl-error.h b/runtime-light/stdlib/rpc/rpc-tl-error.h new file mode 100644 index 0000000000..f3859156ec --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-error.h @@ -0,0 +1,53 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-core/runtime-core.h" +#include "runtime-light/stdlib/rpc/rpc-tl-function.h" + +struct TlRpcError { + int32_t error_code{0}; + string error_msg; + + bool try_fetch() noexcept; + +private: + void fetch_and_skip_header() const noexcept; +}; + +class RpcErrorFactory { +public: + virtual class_instance make_error(const string &error, int32_t error_code) const noexcept = 0; + + class_instance make_error(const char *error, int32_t error_code) const noexcept; + class_instance make_error_from_exception_if_possible() const noexcept; + class_instance fetch_error_if_possible() const noexcept; + + virtual ~RpcErrorFactory() = default; +}; + +namespace tl_rpc_error_impl_ { + +// use template, because _common\Types\rpcResponseError is unknown on runtime compilation +template +struct RpcResponseErrorFactory : public RpcErrorFactory { + RpcResponseErrorFactory() = default; + +private: + class_instance make_error(const string &error, int32_t error_code) const noexcept final { + auto err{make_instance()}; + err.get()->$error = error; + err.get()->$error_code = error_code; + return err; + } +}; + +} // namespace tl_rpc_error_impl_ + +// the definition appears after the TL scheme codegen, during the site build +struct C$VK$TL$_common$Types$rpcResponseError; +using RpcResponseErrorFactory = tl_rpc_error_impl_::RpcResponseErrorFactory; diff --git a/runtime-light/stdlib/rpc/rpc-tl-func-base.h b/runtime-light/stdlib/rpc/rpc-tl-func-base.h new file mode 100644 index 0000000000..c5bc0fd537 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-func-base.h @@ -0,0 +1,34 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-core/allocator/script-allocator-managed.h" +#include "runtime-core/runtime-core.h" +#include "runtime-light/stdlib/rpc/rpc-tl-function.h" + +struct tl_func_base : ScriptAllocatorManaged { + virtual mixed fetch() = 0; + + virtual class_instance typed_fetch() { + // all functions that are called in a typed way override this method with the generated code; + // functions that are not called in a typed way will never call this method + // (it's not a pure virtual method so it's not necessary to generate "return {};" for the untyped functions) + php_critical_error("This function should never be called. Should be overridden in every TL function used in typed mode"); + return {}; + } + + virtual void rpc_server_typed_store([[maybe_unused]] const class_instance &res) { + // all functions annotated with @kphp will override this method with the generated code + php_critical_error("This function should never be called. Should be overridden in every @kphp TL function"); + } + + // every TL function in C++ also has: + // static std::unique_ptr store(const mixed &tl_object); + // static std::unique_ptr typed_store(const C$VK$TL$Functions$thisfunction *tl_object); + // they are not virtual (as they're static), but the implementation is generated for every class + // every one of them creates an instance of itself (fetcher) which is used to do a fetch()/typed_fetch() when the response is received + + virtual ~tl_func_base() = default; +}; diff --git a/runtime-light/stdlib/rpc/rpc-tl-function.h b/runtime-light/stdlib/rpc/rpc-tl-function.h new file mode 100644 index 0000000000..a02a572934 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-function.h @@ -0,0 +1,103 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/algorithms/hashes.h" +#include "common/wrappers/string_view.h" +#include "runtime-core/class-instance/refcountable-php-classes.h" + +struct tl_func_base; + +class ToArrayVisitor; +class CommonMemoryEstimateVisitor; +class InstanceReferencesCountingVisitor; +class InstanceDeepCopyVisitor; +class InstanceDeepDestroyVisitor; + +// The locations of the typed TL related builtin classes that are described in functions.txt +// are hardcoded to the folder/namespace \VK\TL because after the code generation +// C$VK$TL$... should match that layout + +// this interface is implemented by all PHP classes that represent the TL functions (see tl-to-php) +struct C$VK$TL$RpcFunction : abstract_refcountable_php_interface { + virtual const char *get_class() const { + return "VK\\TL\\RpcFunction"; + } + virtual int32_t get_hash() const { + return static_cast(vk::std_hash(vk::string_view(C$VK$TL$RpcFunction::get_class()))); + } + + virtual void accept(ToArrayVisitor &) noexcept {} + virtual void accept(CommonMemoryEstimateVisitor &) noexcept {} + virtual void accept(InstanceReferencesCountingVisitor &) noexcept {} + virtual void accept(InstanceDeepCopyVisitor &) noexcept {} + virtual void accept(InstanceDeepDestroyVisitor &) noexcept {} + + virtual size_t virtual_builtin_sizeof() const noexcept { + return 0; + } + virtual C$VK$TL$RpcFunction *virtual_builtin_clone() const noexcept { + return nullptr; + } + + ~C$VK$TL$RpcFunction() override = default; + virtual std::unique_ptr store() const = 0; +}; + +// every TL function has a class for the result that implements RpcFunctionReturnResult; +// which has ->value of the required type +struct C$VK$TL$RpcFunctionReturnResult : abstract_refcountable_php_interface { + virtual const char *get_class() const { + return "VK\\TL\\RpcFunctionReturnResult"; + } + virtual int32_t get_hash() const { + return static_cast(vk::std_hash(vk::string_view(C$VK$TL$RpcFunctionReturnResult::get_class()))); + } + + virtual void accept(ToArrayVisitor &) noexcept {} + virtual void accept(CommonMemoryEstimateVisitor &) noexcept {} + virtual void accept(InstanceReferencesCountingVisitor &) noexcept {} + virtual void accept(InstanceDeepCopyVisitor &) noexcept {} + virtual void accept(InstanceDeepDestroyVisitor &) noexcept {} + + virtual size_t virtual_builtin_sizeof() const noexcept { + return 0; + } + virtual C$VK$TL$RpcFunctionReturnResult *virtual_builtin_clone() const noexcept { + return nullptr; + } + + ~C$VK$TL$RpcFunctionReturnResult() override = default; +}; + +// function call response — ReqResult from the TL scheme — is a rpcResponseOk|rpcResponseHeader|rpcResponseError; +// if it's rpcResponseOk or rpcResponseHeader, then their bodies can be retrieved by a fetcher that was returned by a store +struct C$VK$TL$RpcResponse : abstract_refcountable_php_interface { + using X = class_instance; + + virtual void accept(ToArrayVisitor &) noexcept {} + virtual void accept(CommonMemoryEstimateVisitor &) noexcept {} + virtual void accept(InstanceReferencesCountingVisitor &) noexcept {} + virtual void accept(InstanceDeepCopyVisitor &) noexcept {} + virtual void accept(InstanceDeepDestroyVisitor &) noexcept {} + + virtual const char *get_class() const { + return "VK\\TL\\RpcResponse"; + } + virtual int32_t get_hash() const { + return static_cast(vk::std_hash(vk::string_view(C$VK$TL$RpcResponse::get_class()))); + } + + virtual size_t virtual_builtin_sizeof() const noexcept { + return 0; + } + virtual C$VK$TL$RpcResponse *virtual_builtin_clone() const noexcept { + return nullptr; + } + + ~C$VK$TL$RpcResponse() override = default; +}; diff --git a/runtime-light/stdlib/rpc/rpc-tl-kphp-request.h b/runtime-light/stdlib/rpc/rpc-tl-kphp-request.h new file mode 100644 index 0000000000..a87ec61030 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-kphp-request.h @@ -0,0 +1,60 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-light/allocator/allocator.h" +#include "runtime-light/stdlib/rpc/rpc-context.h" +#include "runtime-light/stdlib/rpc/rpc-tl-defs.h" +#include "runtime-light/stdlib/rpc/rpc-tl-request.h" + +namespace tl_rpc_request_impl_ { +// use template, because t_ReqResult_ is unknown on runtime compilation +template class t_ReqResult_> +class KphpRpcRequestResult final : public RpcRequestResult { +public: + using RpcRequestResult::RpcRequestResult; + + explicit KphpRpcRequestResult(std::unique_ptr &&result_fetcher) + : RpcRequestResult(true, std::move(result_fetcher)) {} + + class_instance fetch_typed_response() final { + class_instance $response; + t_ReqResult_(tl_exclamation_fetch_wrapper(std::move(result_fetcher))).typed_fetch_to($response); + return $response; + } + + std::unique_ptr extract_untyped_fetcher() final { + php_assert(!"Forbidden to call for typed rpc requests"); + } +}; + +// use template, because t_ReqResult_ is unknown on runtime compilation +template class t_ReqResult_> +class KphpRpcRequest final : public RpcRequest { +public: + using RpcRequest::RpcRequest; + + std::unique_ptr store_request() const final { + // php_assert(CurException.is_null()); + auto &rpc_ctx{RpcComponentContext::get()}; + rpc_ctx.current_query.set_current_tl_function(tl_function_name()); + std::unique_ptr stored_fetcher = storing_function.get()->store(); + rpc_ctx.current_query.reset(); + // if (!CurException.is_null()) { + // CurException = Optional{}; + // return {}; + // } + + return make_unique_on_script_memory>(std::move(stored_fetcher)); + } +}; +} // namespace tl_rpc_request_impl_ + +template +struct t_ReqResult; // the definition appears after the TL scheme codegen, during the site build + +using KphpRpcRequest = tl_rpc_request_impl_::KphpRpcRequest; diff --git a/runtime-light/stdlib/rpc/rpc-tl-query.cpp b/runtime-light/stdlib/rpc/rpc-tl-query.cpp new file mode 100644 index 0000000000..97b709db99 --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-query.cpp @@ -0,0 +1,85 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/rpc/rpc-tl-query.h" + +#include + +#include "runtime-light/stdlib/rpc/rpc-context.h" +#include "runtime-light/utils/php_assert.h" + +void CurrentTlQuery::reset() noexcept { + current_tl_function_name = string{}; +} + +void CurrentTlQuery::set_current_tl_function(const string &tl_function_name) noexcept { + // It can be not empty in the following case: + // 1. Timeout is raised in the middle of serialization (when current TL function is still not reset). + // 2. Then shutdown functions called from timeout. + // 3. They use RPC which finally call set_current_tl_function. + // It will be rewritten by another tl_function_name and work fine + current_tl_function_name = tl_function_name; +} + +void CurrentTlQuery::set_current_tl_function(const class_instance ¤t_query) noexcept { + current_tl_function_name = current_query.get()->tl_function_name; +} + +void CurrentTlQuery::raise_fetching_error(const char *format, ...) const noexcept { + php_assert(!current_tl_function_name.empty()); + + if (/* !CurException.is_null() */ false) { + return; + } + + constexpr size_t BUFF_SZ = 1024; + std::array buff{}; + + va_list args; + va_start(args, format); + int32_t sz = vsnprintf(buff.data(), BUFF_SZ, format, args); + php_assert(sz > 0); + va_end(args); + + string msg = string(buff.data(), static_cast(sz)); + php_warning("Fetching error:\n%s\nIn %s deserializing TL object", msg.c_str(), current_tl_function_name.c_str()); + msg.append(string(" in result of ")).append(current_tl_function_name); + // THROW_EXCEPTION(new_Exception(string{}, 0, msg, -1)); +} + +void CurrentTlQuery::raise_storing_error(const char *format, ...) const noexcept { + if (/*!CurException.is_null()*/ false) { + return; + } + + constexpr size_t BUFF_SZ = 1024; + std::array buff{}; + + va_list args; + va_start(args, format); + int32_t sz = vsnprintf(buff.data(), BUFF_SZ, format, args); + php_assert(sz > 0); + va_end(args); + + string msg = string(buff.data(), static_cast(sz)); + php_warning("Storing error:\n%s\nIn %s serializing TL object", msg.c_str(), + current_tl_function_name.empty() ? "_unknown_" : current_tl_function_name.c_str()); + // THROW_EXCEPTION(new_Exception(string{}, 0, msg, -1)); +} + +void CurrentTlQuery::set_last_stored_tl_function_magic(uint32_t tl_magic) noexcept { + last_stored_tl_function_magic = tl_magic; +} + +uint32_t CurrentTlQuery::get_last_stored_tl_function_magic() const noexcept { + return last_stored_tl_function_magic; +} + +const string &CurrentTlQuery::get_current_tl_function_name() const noexcept { + return current_tl_function_name; +} + +CurrentTlQuery &CurrentTlQuery::get() noexcept { + return RpcComponentContext::get().current_query; +} diff --git a/runtime-light/stdlib/rpc/rpc-tl-query.h b/runtime-light/stdlib/rpc/rpc-tl-query.h new file mode 100644 index 0000000000..b3659b1c3e --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-query.h @@ -0,0 +1,36 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "runtime-core/class-instance/refcountable-php-classes.h" +#include "runtime-core/runtime-core.h" +#include "runtime-light/stdlib/rpc/rpc-tl-request.h" + +struct RpcTlQuery : refcountable_php_classes { + string tl_function_name; + std::unique_ptr result_fetcher; +}; + +struct CurrentTlQuery { + void reset() noexcept; + void set_current_tl_function(const string &tl_function_name) noexcept; + void set_current_tl_function(const class_instance ¤t_query) noexcept; + void raise_fetching_error(const char *format, ...) const noexcept __attribute__((format(printf, 2, 3))); + void raise_storing_error(const char *format, ...) const noexcept __attribute__((format(printf, 2, 3))); + + // called from generated TL serializers (from autogen) + void set_last_stored_tl_function_magic(uint32_t tl_magic) noexcept; + uint32_t get_last_stored_tl_function_magic() const noexcept; + const string &get_current_tl_function_name() const noexcept; + + static CurrentTlQuery &get() noexcept; + +private: + string current_tl_function_name; + uint32_t last_stored_tl_function_magic{0}; +}; diff --git a/runtime-light/stdlib/rpc/rpc-tl-request.cpp b/runtime-light/stdlib/rpc/rpc-tl-request.cpp new file mode 100644 index 0000000000..b46274d7ae --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-request.cpp @@ -0,0 +1,47 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/rpc/rpc-tl-request.h" + +#include "runtime-light/utils/php_assert.h" + +RpcRequestResult::RpcRequestResult(bool is_typed, std::unique_ptr &&result_fetcher) + : is_typed(is_typed) + , result_fetcher(std::move(result_fetcher)) {} + +bool RpcRequestResult::empty() const { + return !result_fetcher; +} + +RpcRequest::RpcRequest(class_instance function) + : storing_function(std::move(function)) {} + +string RpcRequest::tl_function_name() const { + string class_name{storing_function.get()->get_class()}; + const string tl_class_prefix{"\\Functions\\"}; + const auto pos = class_name.find(tl_class_prefix); + if (pos != string::npos) { + class_name = class_name.substr(pos + tl_class_prefix.size(), class_name.size() - (pos + tl_class_prefix.size())); + } + return class_name; +} + +bool RpcRequest::empty() const { + return storing_function.is_null(); +} + +const class_instance &RpcRequest::get_tl_function() const { + return storing_function; +} + +RpcRequestResultUntyped::RpcRequestResultUntyped(std::unique_ptr &&result_fetcher) + : RpcRequestResult(false, std::move(result_fetcher)) {} + +class_instance RpcRequestResultUntyped::fetch_typed_response() { + php_assert(!"Forbidden to call for non typed rpc requests"); +} + +std::unique_ptr RpcRequestResultUntyped::extract_untyped_fetcher() { + return std::move(result_fetcher); +} diff --git a/runtime-light/stdlib/rpc/rpc-tl-request.h b/runtime-light/stdlib/rpc/rpc-tl-request.h new file mode 100644 index 0000000000..7d730a4eed --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-tl-request.h @@ -0,0 +1,56 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-core/allocator/script-allocator-managed.h" +#include "runtime-core/runtime-core.h" +#include "runtime-light/stdlib/rpc/rpc-tl-func-base.h" +#include "runtime-light/stdlib/rpc/rpc-tl-function.h" + +class RpcRequestResult : public ScriptAllocatorManaged { +public: + const bool is_typed{}; + + RpcRequestResult(bool is_typed, std::unique_ptr &&result_fetcher); + + bool empty() const; + + virtual class_instance fetch_typed_response() = 0; + virtual std::unique_ptr extract_untyped_fetcher() = 0; + virtual ~RpcRequestResult() = default; + +protected: + std::unique_ptr result_fetcher; // the store() result +}; + +class RpcRequest { +public: + explicit RpcRequest(class_instance function); + + string tl_function_name() const; + + bool empty() const; + + const class_instance &get_tl_function() const; + + virtual std::unique_ptr store_request() const = 0; + virtual ~RpcRequest() = default; + +protected: + class_instance storing_function; +}; + +class RpcRequestResultUntyped final : public RpcRequestResult { +public: + using RpcRequestResult::RpcRequestResult; + + explicit RpcRequestResultUntyped(std::unique_ptr &&result_fetcher); + + class_instance fetch_typed_response() final; + + std::unique_ptr extract_untyped_fetcher() final; +}; diff --git a/runtime-light/stdlib/stdlib.cmake b/runtime-light/stdlib/stdlib.cmake index 360b50ecf1..b2ea6ac270 100644 --- a/runtime-light/stdlib/stdlib.cmake +++ b/runtime-light/stdlib/stdlib.cmake @@ -5,4 +5,11 @@ prepend(RUNTIME_STDLIB_SRC ${BASE_DIR}/runtime-light/stdlib/ string-functions.cpp variable-handling.cpp superglobals.cpp + rpc/rpc-api.cpp + rpc/rpc-context.cpp + rpc/rpc-extra-headers.cpp + rpc/rpc-extra-info.cpp + rpc/rpc-tl-error.cpp + rpc/rpc-tl-query.cpp + rpc/rpc-tl-request.cpp ) diff --git a/runtime-light/tl/tl-builtins.cpp b/runtime-light/tl/tl-builtins.cpp new file mode 100644 index 0000000000..e10de69d41 --- /dev/null +++ b/runtime-light/tl/tl-builtins.cpp @@ -0,0 +1,204 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/tl/tl-builtins.h" + +void register_tl_storers_table_and_fetcher(const array &gen$ht, tl_fetch_wrapper_ptr gen$t_ReqResult_fetch) { + auto &rpc_mutable_image_state{RpcImageState::get_mutable()}; + rpc_mutable_image_state.tl_storers_ht = gen$ht; + rpc_mutable_image_state.tl_fetch_wrapper = gen$t_ReqResult_fetch; +} + +int32_t tl_parse_save_pos() { + return static_cast(RpcComponentContext::get().fetch_state.pos()); +} + +bool tl_parse_restore_pos(int32_t pos) { + auto &rpc_ctx{RpcComponentContext::get()}; + if (pos < 0 || pos > rpc_ctx.fetch_state.pos()) { + return false; + } + rpc_ctx.fetch_state.reset(static_cast(pos), rpc_ctx.fetch_state.remaining() + rpc_ctx.fetch_state.pos() - pos); + return true; +} + +mixed tl_arr_get(const mixed &arr, const string &str_key, int64_t num_key, int64_t precomputed_hash) { + auto &cur_query{CurrentTlQuery::get()}; + if (!arr.is_array()) { + cur_query.raise_storing_error("Array expected, when trying to access field #%" PRIi64 " : %s", num_key, str_key.c_str()); + return {}; + } + + if (const auto &elem{arr.get_value(num_key)}; !elem.is_null()) { + return elem; + } + if (const auto &elem{precomputed_hash == 0 ? arr.get_value(str_key) : arr.get_value(str_key, precomputed_hash)}; !elem.is_null()) { + return elem; + } + + cur_query.raise_storing_error("Field %s (#%" PRIi64 ") not found", str_key.c_str(), num_key); + return {}; +} + +void store_magic_if_not_bare(uint32_t inner_magic) { + if (static_cast(inner_magic)) { + f$store_int(inner_magic); + } +} + +void fetch_magic_if_not_bare(uint32_t inner_magic, const char *error_msg) { + if (static_cast(inner_magic)) { + const auto actual_magic = static_cast(f$fetch_int()); + if (actual_magic != inner_magic) { + CurrentTlQuery::get().raise_fetching_error("%s\nExpected 0x%08x, but fetched 0x%08x", error_msg, inner_magic, actual_magic); + } + } +} + +void t_Int::store(const mixed &tl_object) { + int32_t v32{prepare_int_for_storing(f$intval(tl_object))}; + f$store_int(v32); +} + +int32_t t_Int::fetch() { + // CHECK_EXCEPTION(return 0); + return f$fetch_int(); +} + +void t_Int::typed_store(const t_Int::PhpType &v) { + int32_t v32{prepare_int_for_storing(v)}; + f$store_int(v32); +} + +void t_Int::typed_fetch_to(t_Int::PhpType &out) { + // CHECK_EXCEPTION(return); + out = f$fetch_int(); +} + +int32_t t_Int::prepare_int_for_storing(int64_t v) { + auto v32 = static_cast(v); + if (is_int32_overflow(v)) { + // TODO + } + return v32; +} + +void t_Long::store(const mixed &tl_object) { + int64_t v64{f$intval(tl_object)}; + f$store_long(v64); +} + +mixed t_Long::fetch() { + // CHECK_EXCEPTION(return mixed()); + return f$fetch_long(); +} + +void t_Long::typed_store(const t_Long::PhpType &v) { + f$store_long(v); +} + +void t_Long::typed_fetch_to(t_Long::PhpType &out) { + // CHECK_EXCEPTION(return); + out = f$fetch_long(); +} + +void t_Double::store(const mixed &tl_object) { + f$store_double(f$floatval(tl_object)); +} + +double t_Double::fetch() { + // CHECK_EXCEPTION(return 0); + return f$fetch_double(); +} + +void t_Double::typed_store(const t_Double::PhpType &v) { + f$store_double(v); +} + +void t_Double::typed_fetch_to(t_Double::PhpType &out) { + // CHECK_EXCEPTION(return); + out = f$fetch_double(); +} + +void t_Float::store(const mixed &tl_object) { + f$store_float(f$floatval(tl_object)); +} + +double t_Float::fetch() { + // CHECK_EXCEPTION(return 0); + return f$fetch_float(); +} + +void t_Float::typed_store(const t_Float::PhpType &v) { + f$store_float(v); +} + +void t_Float::typed_fetch_to(t_Float::PhpType &out) { + // CHECK_EXCEPTION(return); + out = f$fetch_float(); +} + +void t_String::store(const mixed &tl_object) { + f$store_string(f$strval(tl_object)); +} + +string t_String::fetch() { + // CHECK_EXCEPTION(return tl_str_); + return f$fetch_string(); +} + +void t_String::typed_store(const t_String::PhpType &v) { + f$store_string(f$strval(v)); +} + +void t_String::typed_fetch_to(t_String::PhpType &out) { + // CHECK_EXCEPTION(return); + out = f$fetch_string(); +} + +void t_Bool::store(const mixed &tl_object) { + f$store_int(tl_object.to_bool() ? TL_BOOL_TRUE : TL_BOOL_FALSE); +} + +bool t_Bool::fetch() { + // CHECK_EXCEPTION(return false); + const auto magic = static_cast(f$fetch_int()); + switch (magic) { + case TL_BOOL_FALSE: + return false; + case TL_BOOL_TRUE: + return true; + default: { + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Bool: 0x%08x", magic); + return false; + } + } +} + +void t_Bool::typed_store(const t_Bool::PhpType &v) { + f$store_int(v ? TL_BOOL_TRUE : TL_BOOL_FALSE); +} + +void t_Bool::typed_fetch_to(t_Bool::PhpType &out) { + // CHECK_EXCEPTION(return); + const auto magic = static_cast(f$fetch_int()); + if (magic != TL_BOOL_TRUE && magic != TL_BOOL_FALSE) { + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Bool: 0x%08x", magic); + return; + } + out = magic == TL_BOOL_TRUE; +} + +void t_True::store([[maybe_unused]] const mixed &v) {} + +array t_True::fetch() { + return {}; +} + +void t_True::typed_store([[maybe_unused]] const t_True::PhpType &v) {} + +void t_True::typed_fetch_to(t_True::PhpType &out) { + // CHECK_EXCEPTION(return); + out = true; +} diff --git a/runtime-light/tl/tl-builtins.h b/runtime-light/tl/tl-builtins.h new file mode 100644 index 0000000000..91b5b325d4 --- /dev/null +++ b/runtime-light/tl/tl-builtins.h @@ -0,0 +1,560 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "common/php-functions.h" +#include "common/tl/constants/common.h" +#include "runtime-core/runtime-core.h" +#include "runtime-light/stdlib/rpc/rpc-api.h" +#include "runtime-light/stdlib/rpc/rpc-context.h" +#include "runtime-light/stdlib/rpc/rpc-tl-defs.h" +#include "runtime-light/stdlib/rpc/rpc-tl-function.h" +#include "runtime-light/stdlib/rpc/rpc-tl-kphp-request.h" +#include "runtime-light/utils/php_assert.h" + +// TODO: get rid of it here +#define CHECK_EXCEPTION(action) + +void register_tl_storers_table_and_fetcher(const array &gen$ht, tl_fetch_wrapper_ptr gen$t_ReqResult_fetch); + +int32_t tl_parse_save_pos(); + +bool tl_parse_restore_pos(int32_t pos); + +mixed tl_arr_get(const mixed &arr, const string &str_key, int64_t num_key, int64_t precomputed_hash = 0); + +void store_magic_if_not_bare(uint32_t inner_magic); + +void fetch_magic_if_not_bare(uint32_t inner_magic, const char *error_msg); + +template +inline void fetch_raw_vector_T(array &out __attribute__((unused)), int64_t n_elems __attribute__((unused))) { + php_assert(0 && "never called in runtime"); +} + +template<> +inline void fetch_raw_vector_T(array &out, int64_t n_elems) { + fetch_raw_vector_double(out, n_elems); +} + +template +inline void store_raw_vector_T(const array &v __attribute__((unused))) { + php_assert(0 && "never called in runtime"); +} + +template<> +inline void store_raw_vector_T(const array &v) { + store_raw_vector_double(v); +} + +// Wrap into Optional that TL types which PhpType is: +// 1. int, double, string, bool +// 2. array +// These types are not wrapped: +// 1. class_instance +// 2. Optional +// 3. mixed (long in TL scheme or mixed in the phpdoc) UPD: it will be wrapped after the int64_t transition is over + +template +struct need_Optional : vk::is_type_in_list {}; + +template +struct need_Optional> : std::true_type {}; + +enum class FieldAccessType : uint8_t { read, write }; + +// C++14 if constexpr +template +inline const typename SerializerT::PhpType &get_serialization_target_from_optional_field(const OptionalFieldT &v) + requires(need_Optional::value &&ac == FieldAccessType::read) { + return v.val(); +} + +template +inline typename SerializerT::PhpType &get_serialization_target_from_optional_field(OptionalFieldT &v) + requires(need_Optional::value &&ac == FieldAccessType::write) { + return v.ref(); +} + +template +inline OptionalFieldT &get_serialization_target_from_optional_field(OptionalFieldT &v) requires(!need_Optional::value) { + return v; +} + +struct t_Int { + void store(const mixed &tl_object); + int32_t fetch(); + + using PhpType = int64_t; + void typed_store(const PhpType &v); + void typed_fetch_to(PhpType &out); + + static int32_t prepare_int_for_storing(int64_t v); +}; + +struct t_Long { + void store(const mixed &tl_object); + mixed fetch(); + + using PhpType = int64_t; + void typed_store(const PhpType &v); + void typed_fetch_to(PhpType &out); +}; + +struct t_Double { + void store(const mixed &tl_object); + double fetch(); + + using PhpType = double; + void typed_store(const PhpType &v); + void typed_fetch_to(PhpType &out); +}; + +struct t_Float { + void store(const mixed &tl_object); + double fetch(); + + using PhpType = double; + void typed_store(const PhpType &v); + void typed_fetch_to(PhpType &out); +}; + +struct t_String { + void store(const mixed &tl_object); + string fetch(); + + using PhpType = string; + void typed_store(const PhpType &v); + void typed_fetch_to(PhpType &out); +}; + +struct t_Bool { + void store(const mixed &tl_object); + bool fetch(); + + using PhpType = bool; + void typed_store(const PhpType &v); + void typed_fetch_to(PhpType &out); +}; + +struct t_True { + void store(const mixed &v __attribute__((unused))); + array fetch(); + + using PhpType = bool; + void typed_store(const PhpType &v __attribute__((unused))); + void typed_fetch_to(PhpType &out); +}; + +template +struct t_Vector { + T elem_state; + + explicit t_Vector(T param_type) + : elem_state(std::move(param_type)) {} + + void store(const mixed &v) { + const auto &cur_query{CurrentTlQuery::get()}; + + if (!v.is_array()) { + cur_query.raise_storing_error("Expected array, got %s", v.get_type_c_str()); + return; + } + + const array &a = v.as_array(); + int64_t n = v.count(); + f$store_int(n); + for (int64_t i = 0; i < n; ++i) { + if (!a.isset(i)) { + cur_query.raise_storing_error("Vector[%ld] not set", i); + return; + } + store_magic_if_not_bare(inner_magic); + elem_state.store(v.get_value(i)); + } + } + + array fetch() { + // CHECK_EXCEPTION(return array()); + const auto size{f$fetch_int()}; + if (size < 0) { + CurrentTlQuery::get().raise_fetching_error("Vector size is negative"); + return {}; + } + + array res{array_size{size, true}}; + for (int64_t i = 0; i < size; ++i) { + fetch_magic_if_not_bare(inner_magic, "Incorrect magic of inner type of type Vector"); + const mixed &elem{elem_state.fetch()}; + // CHECK_EXCEPTION(return result); + res.push_back(elem); + } + + return res; + } + + using PhpType = array; + using PhpElemT = typename T::PhpType; + + void typed_store(const PhpType &v) { + int64_t n = v.count(); + f$store_int(n); + + if (std::is_same_v && inner_magic == 0 && v.is_vector()) { + store_raw_vector_T(v); + return; + } + + for (int64_t i = 0; i < n; ++i) { + if (!v.isset(i)) { + CurrentTlQuery::get().raise_storing_error("Vector[%" PRIi64 "] not set", i); + return; + } + store_magic_if_not_bare(inner_magic); + elem_state.typed_store(v.get_value(i)); + } + } + + void typed_fetch_to(PhpType &out) { + // CHECK_EXCEPTION(return); + const auto size{f$fetch_int()}; + if (size < 0) { + CurrentTlQuery::get().raise_fetching_error("Vector size is negative"); + return; + } + + out.reserve(size, true); + if (std::is_same_v && inner_magic == 0) { + fetch_raw_vector_T(out, size); + return; + } + + for (int64_t i = 0; i < size; ++i) { + fetch_magic_if_not_bare(inner_magic, "Incorrect magic of inner type of type Vector"); + PhpElemT elem; + elem_state.typed_fetch_to(elem); + out.push_back(std::move(elem)); + // CHECK_EXCEPTION(return); + } + } +}; + +template +struct t_Maybe { + static_assert(!std::is_same_v, "Usage (Maybe True) in TL is forbidden"); + + T elem_state; + + explicit t_Maybe(T param_type) + : elem_state(std::move(param_type)) {} + + // TODO: replace string{...} with constants + void store(const mixed &v) { + const string &name = f$strval(tl_arr_get(v, string{"_"}, 0, string_hash("_", 1))); + if (name == string{"resultFalse"}) { + f$store_int(TL_MAYBE_FALSE); + } else if (name == string{"resultTrue"}) { + f$store_int(TL_MAYBE_TRUE); + store_magic_if_not_bare(inner_magic); + elem_state.store(tl_arr_get(v, string{"result"}, 1, string_hash("result", 6))); + } else { + CurrentTlQuery::get().raise_storing_error("Constructor %s of type Maybe was not found in TL scheme", name.c_str()); + } + } + + mixed fetch() { + // CHECK_EXCEPTION(return mixed()); + const auto magic{static_cast(f$fetch_int())}; + switch (magic) { + case TL_MAYBE_FALSE: + return false; + case TL_MAYBE_TRUE: + fetch_magic_if_not_bare(inner_magic, "Incorrect magic of inner type of type Maybe"); + return elem_state.fetch(); + default: + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Maybe: 0x%08x", magic); + return -1; + } + } + + static constexpr bool inner_needs_Optional = need_Optional::value; + using PhpType = std::conditional_t, typename T::PhpType>; + + static bool has_maybe_value(const PhpType &v) { + return !v.is_null(); + } + + void typed_store(const PhpType &v) { + if (!has_maybe_value(v)) { + f$store_int(TL_MAYBE_FALSE); + } else { + f$store_int(TL_MAYBE_TRUE); + store_magic_if_not_bare(inner_magic); + elem_state.typed_store(get_serialization_target_from_optional_field(v)); + } + } + + void typed_fetch_to(PhpType &out) { + // CHECK_EXCEPTION(return); + const auto magic{static_cast(f$fetch_int())}; + switch (magic) { + case TL_MAYBE_FALSE: + // Wrapped into Optional: array, int64_t, double, string, bool + // Not wrapped: : var, class_instance, Optional + out = PhpType(); + break; + case TL_MAYBE_TRUE: + fetch_magic_if_not_bare(inner_magic, "Incorrect magic of inner type of type Maybe"); + elem_state.typed_fetch_to(get_serialization_target_from_optional_field(out)); + break; + default: + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Maybe: 0x%08x", magic); + return; + } + } +}; + +template +struct tl_Dictionary_impl { + ValueT value_state; + + explicit tl_Dictionary_impl(ValueT value_type) + : value_state(std::move(value_type)) {} + + void store(const mixed &v) { + if (!v.is_array()) { + CurrentTlQuery::get().raise_storing_error("Expected array (dictionary), got something strange"); + return; + } + + int64_t n = v.count(); + f$store_int(n); + for (auto it = v.begin(); it != v.end(); ++it) { + KeyT().store(it.get_key()); + store_magic_if_not_bare(inner_value_magic); + value_state.store(it.get_value()); + } + } + + array fetch() { + // CHECK_EXCEPTION(return array()); + const auto size{f$fetch_int()}; + if (size < 0) { + CurrentTlQuery::get().raise_fetching_error("Dictionary size is negative"); + return {}; + } + + array res{array_size{size, false}}; + for (int64_t i = 0; i < size; ++i) { + const auto &key{KeyT().fetch()}; + fetch_magic_if_not_bare(inner_value_magic, "Incorrect magic of inner type of some Dictionary"); + const mixed &value{value_state.fetch()}; + // CHECK_EXCEPTION(return result); + res.set_value(key, value); + } + return res; + } + + using PhpType = array; + + void typed_store(const PhpType &v) { + int64_t n = v.count(); + f$store_int(n); + + for (auto it = v.begin(); it != v.end(); ++it) { + if constexpr (std::is_same_v) { + KeyT{}.typed_store(it.get_key().to_string()); + } else { + KeyT{}.typed_store(it.get_key().to_int()); + } + store_magic_if_not_bare(inner_value_magic); + value_state.typed_store(it.get_value()); + } + } + + void typed_fetch_to(PhpType &out) { + // CHECK_EXCEPTION(return); + const auto size{f$fetch_int()}; + if (size < 0) { + CurrentTlQuery::get().raise_fetching_error("Dictionary size is negative"); + return; + } + + out.reserve(size, false); + for (int64_t i = 0; i < size; ++i) { + typename KeyT::PhpType key; + KeyT().typed_fetch_to(key); + fetch_magic_if_not_bare(inner_value_magic, "Incorrect magic of inner type of some Dictionary"); + + typename ValueT::PhpType elem; + value_state.typed_fetch_to(elem); + out.set_value(key, std::move(elem)); + // CHECK_EXCEPTION(return); + } + } +}; + +template +using t_Dictionary = tl_Dictionary_impl; + +template +using t_IntKeyDictionary = tl_Dictionary_impl; + +template +using t_LongKeyDictionary = tl_Dictionary_impl; + +template +struct t_Tuple { + T elem_state; + int64_t size; + + t_Tuple(T param_type, int64_t size) + : elem_state(std::move(param_type)) + , size(size) {} + + void store(const mixed &v) { + const auto &cur_query{CurrentTlQuery::get()}; + + if (!v.is_array()) { + cur_query.raise_storing_error("Expected tuple, got %s", v.get_type_c_str()); + return; + } + + const array &a = v.as_array(); + for (int64_t i = 0; i < size; ++i) { + if (!a.isset(i)) { + cur_query.raise_storing_error("Tuple[%ld] not set", i); + return; + } + store_magic_if_not_bare(inner_magic); + elem_state.store(v.get_value(i)); + } + } + + array fetch() { + // CHECK_EXCEPTION(return array()); + array res{array_size{size, true}}; + for (int64_t i = 0; i < size; ++i) { + fetch_magic_if_not_bare(inner_magic, "Incorrect magic of inner type of type Tuple"); + res.push_back(elem_state.fetch()); + } + return res; + } + + using PhpType = array; + + void typed_store(const PhpType &v) { + if (std::is_same_v && inner_magic == 0 && v.is_vector() && v.count() == size) { + store_raw_vector_T(v); + return; + } + + for (int64_t i = 0; i < size; ++i) { + if (!v.isset(i)) { + CurrentTlQuery::get().raise_storing_error("Tuple[%ld] not set", i); + return; + } + store_magic_if_not_bare(inner_magic); + elem_state.typed_store(v.get_value(i)); + } + } + + void typed_fetch_to(PhpType &out) { + // CHECK_EXCEPTION(return); + out.reserve(size, true); + + if (std::is_same_v && inner_magic == 0) { + fetch_raw_vector_T(out, size); + return; + } + + for (int64_t i = 0; i < size; ++i) { + typename T::PhpType elem; + fetch_magic_if_not_bare(inner_magic, "Incorrect magic of inner type of type Tuple"); + elem_state.typed_fetch_to(elem); + out.push_back(std::move(elem)); + // CHECK_EXCEPTION(return); + } + } +}; + +template +struct tl_array { + int64_t size; + T cell; + + tl_array(int64_t size, T cell) + : size(size) + , cell(std::move(cell)) {} + + void store(const mixed &v) { + const auto &cur_query{CurrentTlQuery::get()}; + + if (!v.is_array()) { + cur_query.raise_storing_error("Expected array, got %s", v.get_type_c_str()); + return; + } + + const array &a = v.as_array(); + for (int64_t i = 0; i < size; ++i) { + if (!a.isset(i)) { + cur_query.raise_storing_error("Array[%ld] not set", i); + return; + } + store_magic_if_not_bare(inner_magic); + cell.store(v.get_value(i)); + } + } + + array fetch() { + array result{array_size{size, true}}; + // CHECK_EXCEPTION(return result); + for (int64_t i = 0; i < size; ++i) { + fetch_magic_if_not_bare(inner_magic, "Incorrect magic of inner type of tl array"); + result.push_back(cell.fetch()); + // CHECK_EXCEPTION(return result); + } + return result; + } + + using PhpType = array; + + void typed_store(const PhpType &v) { + if (std::is_same_v && inner_magic == 0 && v.is_vector() && v.count() == size) { + store_raw_vector_T(v); + return; + } + + for (int64_t i = 0; i < size; ++i) { + if (!v.isset(i)) { + CurrentTlQuery::get().raise_storing_error("Array[%ld] not set", i); + return; + } + store_magic_if_not_bare(inner_magic); + cell.typed_store(v.get_value(i)); + } + } + + void typed_fetch_to(PhpType &out) { + // CHECK_EXCEPTION(return); + out.reserve(size, true); + + if (std::is_same_v && inner_magic == 0) { + fetch_raw_vector_T(out, size); + return; + } + + for (int64_t i = 0; i < size; ++i) { + typename T::PhpType elem; + fetch_magic_if_not_bare(inner_magic, "Incorrect magic of inner type of tl array"); + cell.typed_fetch_to(elem); + out.push_back(std::move(elem)); + // CHECK_EXCEPTION(return); + } + } +}; diff --git a/runtime-light/tl/tl.cmake b/runtime-light/tl/tl.cmake new file mode 100644 index 0000000000..f80d755753 --- /dev/null +++ b/runtime-light/tl/tl.cmake @@ -0,0 +1,3 @@ +prepend(RUNTIME_TL_SRC ${BASE_DIR}/runtime-light/tl/ + tl-builtins.cpp +) diff --git a/runtime-light/utils/concepts.h b/runtime-light/utils/concepts.h new file mode 100644 index 0000000000..963a9c6b95 --- /dev/null +++ b/runtime-light/utils/concepts.h @@ -0,0 +1,10 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +template +concept standard_layout = std::is_standard_layout_v; diff --git a/runtime/rpc.cpp b/runtime/rpc.cpp index 0568326b81..d3f0ca9cd7 100644 --- a/runtime/rpc.cpp +++ b/runtime/rpc.cpp @@ -759,7 +759,7 @@ int64_t rpc_send_impl(const class_instance &conn, double timeou double send_timestamp = std::chrono::duration{std::chrono::system_clock::now().time_since_epoch()}.count(); cur->resumable_id = register_forked_resumable(new rpc_resumable(q_id)); - cur->function_magic = CurrentProcessingQuery::get().get_last_stored_tl_function_magic(); + cur->function_magic = CurrentTlQuery::get().get_last_stored_tl_function_magic(); cur->actor_or_port = conn.get()->actor_id > 0 ? conn.get()->actor_id : -conn.get()->port; cur->timer = nullptr; @@ -1128,21 +1128,21 @@ bool try_fetch_rpc_error(array &out_if_error) { class_instance store_function(const mixed &tl_object) { php_assert(CurException.is_null()); if (!tl_object.is_array()) { - CurrentProcessingQuery::get().raise_storing_error("Not an array passed to function rpc_tl_query"); + CurrentTlQuery::get().raise_storing_error("Not an array passed to function rpc_tl_query"); return {}; } string fun_name = tl_arr_get(tl_object, tl_str_underscore, 0).to_string(); if (!tl_storers_ht.has_key(fun_name)) { - CurrentProcessingQuery::get().raise_storing_error("Function \"%s\" not found in tl-scheme", fun_name.c_str()); + CurrentTlQuery::get().raise_storing_error("Function \"%s\" not found in tl-scheme", fun_name.c_str()); return {}; } class_instance rpc_query; rpc_query.alloc(); rpc_query.get()->tl_function_name = fun_name; - CurrentProcessingQuery::get().set_current_tl_function(fun_name); + CurrentTlQuery::get().set_current_tl_function(fun_name); const auto &untyped_storer = tl_storers_ht.get_value(fun_name); rpc_query.get()->result_fetcher = make_unique_on_script_memory(untyped_storer(tl_object)); - CurrentProcessingQuery::get().reset(); + CurrentTlQuery::get().reset(); return rpc_query; } @@ -1152,11 +1152,11 @@ array fetch_function(const class_instance &rpc_query) { return new_tl_object; // this object carries an error (see tl_fetch_error()) } php_assert(!rpc_query.is_null()); - CurrentProcessingQuery::get().set_current_tl_function(rpc_query); + CurrentTlQuery::get().set_current_tl_function(rpc_query); auto stored_fetcher = rpc_query.get()->result_fetcher->extract_untyped_fetcher(); php_assert(stored_fetcher); new_tl_object = tl_fetch_wrapper(std::move(stored_fetcher)); - CurrentProcessingQuery::get().reset(); + CurrentTlQuery::get().reset(); if (!CurException.is_null()) { array result = tl_fetch_error(CurException->$message, TL_ERROR_SYNTAX); CurException = Optional{}; @@ -1480,7 +1480,7 @@ static void reset_rpc_global_vars() { void init_rpc_lib() { php_assert (timeout_wakeup_id != -1); - CurrentProcessingQuery::get().reset(); + CurrentTlQuery::get().reset(); RpcPendingQueries::get().hard_reset(); CurrentRpcServerQuery::get().reset(); reset_rpc_global_vars(); @@ -1500,7 +1500,7 @@ void init_rpc_lib() { void free_rpc_lib() { reset_rpc_global_vars(); RpcPendingQueries::get().hard_reset(); - CurrentProcessingQuery::get().reset(); + CurrentTlQuery::get().reset(); } int64_t f$rpc_queue_create() { diff --git a/runtime/tl/rpc_request.h b/runtime/tl/rpc_request.h index b15f6ddd2d..3fd340e85a 100644 --- a/runtime/tl/rpc_request.h +++ b/runtime/tl/rpc_request.h @@ -106,9 +106,9 @@ class KphpRpcRequest final : public RpcRequest { std::unique_ptr store_request() const final { php_assert(CurException.is_null()); - CurrentProcessingQuery::get().set_current_tl_function(tl_function_name()); + CurrentTlQuery::get().set_current_tl_function(tl_function_name()); std::unique_ptr stored_fetcher = storing_function_.get()->store(); - CurrentProcessingQuery::get().reset(); + CurrentTlQuery::get().reset(); if (!CurException.is_null()) { CurException = Optional{}; return {}; diff --git a/runtime/tl/rpc_tl_query.cpp b/runtime/tl/rpc_tl_query.cpp index c684ae26e6..945d643ff8 100644 --- a/runtime/tl/rpc_tl_query.cpp +++ b/runtime/tl/rpc_tl_query.cpp @@ -24,11 +24,11 @@ void RpcPendingQueries::hard_reset() { hard_reset_var(queries_); } -void CurrentProcessingQuery::reset() { +void CurrentTlQuery::reset() { current_tl_function_name_ = string(); } -void CurrentProcessingQuery::set_current_tl_function(const string &tl_function_name) { +void CurrentTlQuery::set_current_tl_function(const string &tl_function_name) { // It can be not empty in the following case: // 1. Timeout is raised in the middle of serialization (when current TL function is still not reset). // 2. Then shutdown functions called from timeout. @@ -37,11 +37,11 @@ void CurrentProcessingQuery::set_current_tl_function(const string &tl_function_n current_tl_function_name_ = tl_function_name; } -void CurrentProcessingQuery::set_current_tl_function(const class_instance ¤t_query) { +void CurrentTlQuery::set_current_tl_function(const class_instance ¤t_query) { current_tl_function_name_ = current_query.get()->tl_function_name; } -void CurrentProcessingQuery::raise_fetching_error(const char *format, ...) { +void CurrentTlQuery::raise_fetching_error(const char *format, ...) { php_assert(!current_tl_function_name_.empty()); if (CurException.is_null()) { constexpr size_t BUFF_SZ = 1024; @@ -58,7 +58,7 @@ void CurrentProcessingQuery::raise_fetching_error(const char *format, ...) { } } -void CurrentProcessingQuery::raise_storing_error(const char *format, ...) { +void CurrentTlQuery::raise_storing_error(const char *format, ...) { const char *function_name = current_tl_function_name_.empty() ? "_unknown_" : current_tl_function_name_.c_str(); if (CurException.is_null()) { constexpr size_t BUFF_SZ = 1024; diff --git a/runtime/tl/rpc_tl_query.h b/runtime/tl/rpc_tl_query.h index 39d3b5fa7f..1d979d8549 100644 --- a/runtime/tl/rpc_tl_query.h +++ b/runtime/tl/rpc_tl_query.h @@ -37,10 +37,10 @@ class RpcPendingQueries { array> queries_; }; -class CurrentProcessingQuery { +class CurrentTlQuery { public: - static CurrentProcessingQuery &get() { - static CurrentProcessingQuery context; + static CurrentTlQuery &get() { + static CurrentTlQuery context; return context; } diff --git a/runtime/tl/tl_builtins.h b/runtime/tl/tl_builtins.h index d134c58213..417b331553 100644 --- a/runtime/tl/tl_builtins.h +++ b/runtime/tl/tl_builtins.h @@ -45,7 +45,7 @@ using tl_storer_ptr = std::unique_ptr(*)(const mixed &); inline mixed tl_arr_get(const mixed &arr, const string &str_key, int64_t num_key, int64_t precomputed_hash = 0) { if (!arr.is_array()) { - CurrentProcessingQuery::get().raise_storing_error("Array expected, when trying to access field #%" PRIi64 " : %s", num_key, str_key.c_str()); + CurrentTlQuery::get().raise_storing_error("Array expected, when trying to access field #%" PRIi64 " : %s", num_key, str_key.c_str()); return {}; } const mixed &num_v = arr.get_value(num_key); @@ -56,7 +56,7 @@ inline mixed tl_arr_get(const mixed &arr, const string &str_key, int64_t num_key if (!str_v.is_null()) { return str_v; } - CurrentProcessingQuery::get().raise_storing_error("Field %s (#%" PRIi64 ") not found", str_key.c_str(), num_key); + CurrentTlQuery::get().raise_storing_error("Field %s (#%" PRIi64 ") not found", str_key.c_str(), num_key); return {}; } @@ -70,7 +70,7 @@ inline void fetch_magic_if_not_bare(unsigned int inner_magic, const char *error_ if (inner_magic) { auto actual_magic = static_cast(rpc_fetch_int()); if (actual_magic != inner_magic) { - CurrentProcessingQuery::get().raise_fetching_error("%s\nExpected 0x%08x, but fetched 0x%08x", error_msg, inner_magic, actual_magic); + CurrentTlQuery::get().raise_fetching_error("%s\nExpected 0x%08x, but fetched 0x%08x", error_msg, inner_magic, actual_magic); } } } @@ -164,10 +164,10 @@ struct t_Int { auto v32 = static_cast(v); if (unlikely(is_int32_overflow(v))) { if (fail_rpc_on_int32_overflow) { - CurrentProcessingQuery::get().raise_storing_error("Got int32 overflow with value '%" PRIi64 "'. Serialization will fail.", v); + CurrentTlQuery::get().raise_storing_error("Got int32 overflow with value '%" PRIi64 "'. Serialization will fail.", v); } else { php_warning("Got int32 overflow on storing %s: the value '%" PRIi64 "' will be casted to '%d'. Serialization will succeed.", - CurrentProcessingQuery::get().get_current_tl_function_name().c_str(), v, v32); + CurrentTlQuery::get().get_current_tl_function_name().c_str(), v, v32); } } return v32; @@ -276,7 +276,7 @@ struct t_Bool { case TL_BOOL_TRUE: return true; default: { - CurrentProcessingQuery::get().raise_fetching_error("Incorrect magic of type Bool: 0x%08x", magic); + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Bool: 0x%08x", magic); return -1; } } @@ -292,7 +292,7 @@ struct t_Bool { CHECK_EXCEPTION(return); auto magic = static_cast(rpc_fetch_int()); if (magic != TL_BOOL_TRUE && magic != TL_BOOL_FALSE) { - CurrentProcessingQuery::get().raise_fetching_error("Incorrect magic of type Bool: 0x%08x", magic); + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Bool: 0x%08x", magic); return; } out = magic == TL_BOOL_TRUE; @@ -325,7 +325,7 @@ struct t_Vector { void store(const mixed &v) { if (!v.is_array()) { - CurrentProcessingQuery::get().raise_storing_error("Expected array, got %s", v.get_type_c_str()); + CurrentTlQuery::get().raise_storing_error("Expected array, got %s", v.get_type_c_str()); return; } const array &a = v.as_array(); @@ -333,7 +333,7 @@ struct t_Vector { f$store_int(n); for (int64_t i = 0; i < n; ++i) { if (!a.isset(i)) { - CurrentProcessingQuery::get().raise_storing_error("Vector[%ld] not set", i); + CurrentTlQuery::get().raise_storing_error("Vector[%ld] not set", i); return; } store_magic_if_not_bare(inner_magic); @@ -345,7 +345,7 @@ struct t_Vector { CHECK_EXCEPTION(return array()); int n = rpc_fetch_int(); if (n < 0) { - CurrentProcessingQuery::get().raise_fetching_error("Vector size is negative"); + CurrentTlQuery::get().raise_fetching_error("Vector size is negative"); return {}; } array result(array_size(std::min(n, 10000), true)); @@ -372,7 +372,7 @@ struct t_Vector { for (int64_t i = 0; i < n; ++i) { if (!v.isset(i)) { - CurrentProcessingQuery::get().raise_storing_error("Vector[%" PRIi64 "] not set", i); + CurrentTlQuery::get().raise_storing_error("Vector[%" PRIi64 "] not set", i); return; } store_magic_if_not_bare(inner_magic); @@ -384,7 +384,7 @@ struct t_Vector { CHECK_EXCEPTION(return); int n = rpc_fetch_int(); if (n < 0) { - CurrentProcessingQuery::get().raise_fetching_error("Vector size is negative"); + CurrentTlQuery::get().raise_fetching_error("Vector size is negative"); return; } out.reserve(n, true); @@ -422,7 +422,7 @@ struct t_Maybe { store_magic_if_not_bare(inner_magic); elem_state.store(tl_arr_get(v, tl_str_result, 1, tl_str_result_hash)); } else { - CurrentProcessingQuery::get().raise_storing_error("Constructor %s of type Maybe was not found in TL scheme", name.c_str()); + CurrentTlQuery::get().raise_storing_error("Constructor %s of type Maybe was not found in TL scheme", name.c_str()); return; } } @@ -439,7 +439,7 @@ struct t_Maybe { return elem_state.fetch(); } default: { - CurrentProcessingQuery::get().raise_fetching_error("Incorrect magic of type Maybe: 0x%08x", magic); + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Maybe: 0x%08x", magic); return -1; } } @@ -478,7 +478,7 @@ struct t_Maybe { break; } default: { - CurrentProcessingQuery::get().raise_fetching_error("Incorrect magic of type Maybe: 0x%08x", magic); + CurrentTlQuery::get().raise_fetching_error("Incorrect magic of type Maybe: 0x%08x", magic); return; } } @@ -494,7 +494,7 @@ struct tl_Dictionary_impl { void store(const mixed &v) { if (!v.is_array()) { - CurrentProcessingQuery::get().raise_storing_error("Expected array (dictionary), got something strange"); + CurrentTlQuery::get().raise_storing_error("Expected array (dictionary), got something strange"); return; } int64_t n = v.count(); @@ -511,7 +511,7 @@ struct tl_Dictionary_impl { array result; int32_t n = rpc_fetch_int(); if (n < 0) { - CurrentProcessingQuery::get().raise_fetching_error("Dictionary size is negative"); + CurrentTlQuery::get().raise_fetching_error("Dictionary size is negative"); return result; } for (int32_t i = 0; i < n; ++i) { @@ -544,7 +544,7 @@ struct tl_Dictionary_impl { CHECK_EXCEPTION(return); int32_t n = rpc_fetch_int(); if (n < 0) { - CurrentProcessingQuery::get().raise_fetching_error("Dictionary size is negative"); + CurrentTlQuery::get().raise_fetching_error("Dictionary size is negative"); return; } for (int32_t i = 0; i < n; ++i) { @@ -579,13 +579,13 @@ struct t_Tuple { void store(const mixed &v) { if (!v.is_array()) { - CurrentProcessingQuery::get().raise_storing_error("Expected tuple, got %s", v.get_type_c_str()); + CurrentTlQuery::get().raise_storing_error("Expected tuple, got %s", v.get_type_c_str()); return; } const array &a = v.as_array(); for (int64_t i = 0; i < size; ++i) { if (!a.isset(i)) { - CurrentProcessingQuery::get().raise_storing_error("Tuple[%ld] not set", i); + CurrentTlQuery::get().raise_storing_error("Tuple[%ld] not set", i); return; } store_magic_if_not_bare(inner_magic); @@ -613,7 +613,7 @@ struct t_Tuple { for (int64_t i = 0; i < size; ++i) { if (!v.isset(i)) { - CurrentProcessingQuery::get().raise_storing_error("Tuple[%ld] not set", i); + CurrentTlQuery::get().raise_storing_error("Tuple[%ld] not set", i); return; } store_magic_if_not_bare(inner_magic); @@ -651,13 +651,13 @@ struct tl_array { void store(const mixed &v) { if (!v.is_array()) { - CurrentProcessingQuery::get().raise_storing_error("Expected array, got %s", v.get_type_c_str()); + CurrentTlQuery::get().raise_storing_error("Expected array, got %s", v.get_type_c_str()); return; } const array &a = v.as_array(); for (int64_t i = 0; i < size; ++i) { if (!a.isset(i)) { - CurrentProcessingQuery::get().raise_storing_error("Array[%ld] not set", i); + CurrentTlQuery::get().raise_storing_error("Array[%ld] not set", i); return; } store_magic_if_not_bare(inner_magic); @@ -686,7 +686,7 @@ struct tl_array { for (int64_t i = 0; i < size; ++i) { if (!v.isset(i)) { - CurrentProcessingQuery::get().raise_storing_error("Array[%ld] not set", i); + CurrentTlQuery::get().raise_storing_error("Array[%ld] not set", i); return; } store_magic_if_not_bare(inner_magic); diff --git a/runtime/typed_rpc.cpp b/runtime/typed_rpc.cpp index 867ade841c..e327619f6a 100644 --- a/runtime/typed_rpc.cpp +++ b/runtime/typed_rpc.cpp @@ -59,9 +59,9 @@ class typed_rpc_tl_query_result_one_resumable : public Resumable { RETURN(error_factory_.make_error(last_rpc_error_message_get(), last_rpc_error_code_get())); } - CurrentProcessingQuery::get().set_current_tl_function(query_); + CurrentTlQuery::get().set_current_tl_function(query_); auto rpc_result = fetch_result(std::move(query_.get()->result_fetcher), error_factory_); - CurrentProcessingQuery::get().reset(); + CurrentTlQuery::get().reset(); rpc_parse_restore_previous(); RETURN(rpc_result); RESUMABLE_END @@ -243,7 +243,7 @@ array> typed_rpc_tl_query_result_synchronous } void free_typed_rpc_lib() { - CurrentProcessingQuery::get().reset(); + CurrentTlQuery::get().reset(); RpcPendingQueries::get().hard_reset(); CurrentRpcServerQuery::get().reset(); } diff --git a/tests/cpp/runtime/memory_resource/details/memory_chunk_tree-test.cpp b/tests/cpp/runtime/memory_resource/details/memory_chunk_tree-test.cpp index d283440fd3..da5b377700 100644 --- a/tests/cpp/runtime/memory_resource/details/memory_chunk_tree-test.cpp +++ b/tests/cpp/runtime/memory_resource/details/memory_chunk_tree-test.cpp @@ -1,5 +1,7 @@ #include +#include #include +#include #include #include "runtime-core/memory-resource/details/memory_chunk_tree.h" @@ -12,7 +14,7 @@ TEST(memory_chunk_tree_test, empty) { ASSERT_FALSE(mem_chunk_tree.extract(1)); ASSERT_FALSE(mem_chunk_tree.extract(9999)); - memory_resource::details::memory_ordered_chunk_list mem_list{nullptr}; + memory_resource::details::memory_ordered_chunk_list mem_list{nullptr, nullptr}; mem_chunk_tree.flush_to(mem_list); ASSERT_FALSE(mem_list.flush()); } @@ -20,9 +22,9 @@ TEST(memory_chunk_tree_test, empty) { TEST(memory_chunk_tree_test, hard_reset) { memory_resource::details::memory_chunk_tree mem_chunk_tree; - char some_memory[1024]; - mem_chunk_tree.insert(some_memory, 100); - mem_chunk_tree.insert(some_memory + 200, 150); + std::array some_memory{}; + mem_chunk_tree.insert(some_memory.data(), 100); + mem_chunk_tree.insert(some_memory.data() + 200, 150); mem_chunk_tree.hard_reset(); ASSERT_FALSE(mem_chunk_tree.extract_smallest()); @@ -32,34 +34,31 @@ TEST(memory_chunk_tree_test, hard_reset) { TEST(memory_chunk_tree_test, insert_extract) { memory_resource::details::memory_chunk_tree mem_chunk_tree; - char some_memory[1024 * 1024]; + std::array(1024 * 1024)> some_memory{}; const auto chunk_sizes = prepare_test_sizes(); const auto chunk_offsets = make_offsets(chunk_sizes); for (size_t i = 0; i < chunk_sizes.size(); ++i) { - mem_chunk_tree.insert(some_memory + chunk_offsets[i], chunk_sizes[i]); + mem_chunk_tree.insert(some_memory.data() + chunk_offsets[i], chunk_sizes[i]); } size_t extracted = 0; for (int i = 0; i < 2; ++i, ++extracted) { auto *mem = mem_chunk_tree.extract(99); ASSERT_TRUE(mem); - ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), - memory_resource::details::align_for_chunk(99)); + ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), memory_resource::details::align_for_chunk(99)); } for (int i = 0; i < 3; ++i, ++extracted) { auto *mem = mem_chunk_tree.extract(99); ASSERT_TRUE(mem); - ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), - memory_resource::details::align_for_chunk(100)); + ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), memory_resource::details::align_for_chunk(100)); } for (int i = 0; i < 2; ++i, ++extracted) { auto *mem = mem_chunk_tree.extract(112); ASSERT_TRUE(mem); - ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), - memory_resource::details::align_for_chunk(112)); + ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), memory_resource::details::align_for_chunk(112)); } auto *mem = mem_chunk_tree.extract(250); @@ -71,14 +70,12 @@ TEST(memory_chunk_tree_test, insert_extract) { mem = mem_chunk_tree.extract(1); ++extracted; ASSERT_TRUE(mem); - ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), - memory_resource::details::align_for_chunk(42)); + ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), memory_resource::details::align_for_chunk(42)); mem = mem_chunk_tree.extract(42); ++extracted; ASSERT_TRUE(mem); - ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), - memory_resource::details::align_for_chunk(45)); + ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), memory_resource::details::align_for_chunk(45)); size_t prev_size = 0; for (; extracted != chunk_sizes.size(); ++extracted) { @@ -92,20 +89,19 @@ TEST(memory_chunk_tree_test, insert_extract) { ASSERT_FALSE(mem); } - TEST(memory_chunk_tree_test, insert_extract_smallest) { memory_resource::details::memory_chunk_tree mem_chunk_tree; - char some_memory[1024 * 1024]; + std::array(1024 * 1024)> some_memory{}; auto chunk_sizes = prepare_test_sizes(); const auto chunk_offsets = make_offsets(chunk_sizes); for (size_t i = 0; i < chunk_sizes.size(); ++i) { - mem_chunk_tree.insert(some_memory + chunk_offsets[i], chunk_sizes[i]); + mem_chunk_tree.insert(some_memory.data() + chunk_offsets[i], chunk_sizes[i]); } std::sort(chunk_sizes.begin(), chunk_sizes.end()); - for (auto chunk_size: chunk_sizes) { + for (auto chunk_size : chunk_sizes) { auto *mem = mem_chunk_tree.extract_smallest(); ASSERT_TRUE(mem); ASSERT_EQ(memory_resource::details::memory_chunk_tree::get_chunk_size(mem), chunk_size); @@ -118,22 +114,22 @@ TEST(memory_chunk_tree_test, insert_extract_smallest) { TEST(memory_chunk_tree_test, flush_to) { memory_resource::details::memory_chunk_tree mem_chunk_tree; - char some_memory[1024 * 1024]; + std::array(1024 * 1024)> some_memory{}; const auto chunk_sizes = prepare_test_sizes(); const auto chunk_offsets = make_offsets(chunk_sizes); for (size_t i = 0; i < chunk_sizes.size(); ++i) { - mem_chunk_tree.insert(some_memory + chunk_offsets[i], chunk_sizes[i]); + mem_chunk_tree.insert(some_memory.data() + chunk_offsets[i], chunk_sizes[i]); } - memory_resource::details::memory_ordered_chunk_list ordered_list{some_memory}; + memory_resource::details::memory_ordered_chunk_list ordered_list{some_memory.data(), some_memory.data() + some_memory.size()}; mem_chunk_tree.flush_to(ordered_list); ASSERT_FALSE(mem_chunk_tree.extract_smallest()); auto *first_node = ordered_list.flush(); ASSERT_TRUE(first_node); ASSERT_EQ(first_node->size(), chunk_offsets.back()); - ASSERT_EQ(reinterpret_cast(first_node), some_memory); + ASSERT_EQ(reinterpret_cast(first_node), some_memory.data()); ASSERT_FALSE(ordered_list.get_next(first_node)); } diff --git a/tests/cpp/runtime/memory_resource/details/memory_ordered_chunk_list-test.cpp b/tests/cpp/runtime/memory_resource/details/memory_ordered_chunk_list-test.cpp index 9dc1d58687..b29acbfb3d 100644 --- a/tests/cpp/runtime/memory_resource/details/memory_ordered_chunk_list-test.cpp +++ b/tests/cpp/runtime/memory_resource/details/memory_ordered_chunk_list-test.cpp @@ -1,30 +1,31 @@ #include +#include +#include #include #include - #include "runtime-core/memory-resource/details/memory_ordered_chunk_list.h" #include "runtime-core/memory-resource/monotonic_buffer_resource.h" #include "tests/cpp/runtime/memory_resource/details/test-helpers.h" TEST(memory_ordered_chunk_list_test, empty) { - memory_resource::details::memory_ordered_chunk_list mem_list{nullptr}; + memory_resource::details::memory_ordered_chunk_list mem_list{nullptr, nullptr}; ASSERT_FALSE(mem_list.flush()); } TEST(memory_ordered_chunk_list_test, add_memory_and_flush_to) { - char some_memory[1024 * 1024]; + std::array(1024 * 1024)> some_memory{}; const auto chunk_sizes = prepare_test_sizes(); const auto chunk_offsets = make_offsets(chunk_sizes); - memory_resource::details::memory_ordered_chunk_list ordered_list{some_memory}; + memory_resource::details::memory_ordered_chunk_list ordered_list{some_memory.data(), some_memory.data() + some_memory.size()}; for (size_t i = 0; i < chunk_sizes.size(); ++i) { - ordered_list.add_memory(some_memory + chunk_offsets[i], chunk_sizes[i]); + ordered_list.add_memory(some_memory.data() + chunk_offsets[i], chunk_sizes[i]); } constexpr size_t gap1 = memory_resource::details::align_for_chunk(510250); ASSERT_GT(gap1, chunk_offsets.back()); - char *some_memory_with_gap1 = some_memory + gap1; + char *some_memory_with_gap1 = some_memory.data() + gap1; for (size_t i = 0; i < chunk_sizes.size(); ++i) { ordered_list.add_memory(some_memory_with_gap1 + chunk_offsets[i], chunk_sizes[i]); } @@ -32,7 +33,7 @@ TEST(memory_ordered_chunk_list_test, add_memory_and_flush_to) { constexpr size_t gap2 = memory_resource::details::align_for_chunk(235847); ASSERT_GT(gap2, chunk_offsets.back()); ASSERT_GT(gap1, chunk_offsets.back() + gap2); - char *some_memory_with_gap2 = some_memory + gap2; + char *some_memory_with_gap2 = some_memory.data() + gap2; for (size_t i = 0; i < chunk_sizes.size(); ++i) { ordered_list.add_memory(some_memory_with_gap2 + chunk_offsets[i], chunk_sizes[i]); } @@ -50,7 +51,7 @@ TEST(memory_ordered_chunk_list_test, add_memory_and_flush_to) { auto *no_gap_node = ordered_list.get_next(gap2_node); ASSERT_TRUE(no_gap_node); ASSERT_EQ(no_gap_node->size(), chunk_offsets.back()); - ASSERT_EQ(reinterpret_cast(no_gap_node), some_memory); + ASSERT_EQ(reinterpret_cast(no_gap_node), some_memory.data()); ASSERT_FALSE(ordered_list.get_next(no_gap_node)); } @@ -60,15 +61,14 @@ TEST(memory_ordered_chunk_list_test, add_random) { std::mt19937 gen(rd()); std::uniform_int_distribution dis(8, 63); - constexpr size_t memory_size = 1024 * 1024; - char some_memory[memory_size]; + std::array(1024 * 1024)> some_memory{}; memory_resource::monotonic_buffer_resource mem_resource; - mem_resource.init(some_memory, memory_size); + mem_resource.init(some_memory.data(), some_memory.size()); constexpr size_t total_chunks = 16 * 1024; std::array, total_chunks> chunks; - memory_resource::details::memory_ordered_chunk_list ordered_list{some_memory}; + memory_resource::details::memory_ordered_chunk_list ordered_list{some_memory.data(), some_memory.data() + some_memory.size()}; for (auto &chunk : chunks) { size_t mem_size = dis(gen); diff --git a/tests/cpp/runtime/memory_resource/unsynchronized_pool_resource-test.cpp b/tests/cpp/runtime/memory_resource/unsynchronized_pool_resource-test.cpp index 81dc799f2e..969fba48df 100644 --- a/tests/cpp/runtime/memory_resource/unsynchronized_pool_resource-test.cpp +++ b/tests/cpp/runtime/memory_resource/unsynchronized_pool_resource-test.cpp @@ -254,7 +254,7 @@ TEST(unsynchronized_pool_resource_test, test_auto_defragmentation_huge_piece) { TEST(unsynchronized_pool_resource_test, test_auto_defragmentation_small_piece) { - std::array some_memory{}; + std::array(1024 * 32)> some_memory{}; memory_resource::unsynchronized_pool_resource resource; resource.init(some_memory.data(), some_memory.size()); @@ -292,4 +292,46 @@ TEST(unsynchronized_pool_resource_test, test_auto_defragmentation_small_piece) { ASSERT_EQ(mem_stats.small_memory_pieces, 0); resource.deallocate(mem64, 64); -} \ No newline at end of file +} + +TEST(unsynchronized_pool_resource_test, test_auto_defragmentation_with_extra_memory_pool) { + std::array(18 * 1024)> some_memory{}; + memory_resource::unsynchronized_pool_resource resource; + resource.init(some_memory.data(), some_memory.size()); + + std::array(32 * 1024) + sizeof(memory_resource::extra_memory_pool)> extra_memory{}; + resource.add_extra_memory(new (extra_memory.data()) memory_resource::extra_memory_pool{extra_memory.size()}); + + std::array pieces1024{}; + for (auto &mem : pieces1024) { + mem = resource.allocate(1024); + } + for (auto *mem : pieces1024) { + resource.deallocate(mem, 1024); + } + + // got fragmentation + auto mem_stats = resource.get_memory_stats(); + ASSERT_EQ(mem_stats.real_memory_used, some_memory.size() - 1024); + ASSERT_EQ(mem_stats.memory_used, 0); + ASSERT_EQ(mem_stats.max_real_memory_used, some_memory.size()); + ASSERT_EQ(mem_stats.max_memory_used, some_memory.size() + extra_memory.size() - sizeof(memory_resource::extra_memory_pool)); + ASSERT_EQ(mem_stats.memory_limit, some_memory.size()); + ASSERT_EQ(mem_stats.defragmentation_calls, 0); + ASSERT_EQ(mem_stats.huge_memory_pieces, 0); + ASSERT_EQ(mem_stats.small_memory_pieces, 49); + + // auto defragmentation + void *mem2048 = resource.allocate(2048); + mem_stats = resource.get_memory_stats(); + ASSERT_EQ(mem_stats.real_memory_used, 2048); + ASSERT_EQ(mem_stats.memory_used, 2048); + ASSERT_EQ(mem_stats.max_real_memory_used, some_memory.size()); + ASSERT_EQ(mem_stats.max_memory_used, some_memory.size() + extra_memory.size() - sizeof(memory_resource::extra_memory_pool)); + ASSERT_EQ(mem_stats.memory_limit, some_memory.size()); + ASSERT_EQ(mem_stats.defragmentation_calls, 1); + ASSERT_EQ(mem_stats.huge_memory_pieces, 0); + ASSERT_EQ(mem_stats.small_memory_pieces, 0); + + resource.deallocate(mem2048, 2048); +} From 2ac101888ccb4533643de186f4d5511a0787946f Mon Sep 17 00:00:00 2001 From: Vadim Sadokhov <65451602+astrophysik@users.noreply.github.com> Date: Fri, 5 Jul 2024 20:14:31 +0300 Subject: [PATCH 44/89] sync k2 header with header in runtime light (#1035) --- compiler/code-gen/files/init-scripts.cpp | 5 ++++- compiler/compiler-settings.cpp | 12 ++++++++++++ compiler/compiler-settings.h | 2 ++ compiler/kphp2cpp.cpp | 4 +++- runtime-light/header.h | 14 +++++++++----- runtime-light/stdlib/string-functions.cpp | 2 +- tests/k2-components/oom_loop.php | 3 +++ 7 files changed, 34 insertions(+), 8 deletions(-) diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index 1a7f1b0a33..388e2e6748 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -283,7 +283,10 @@ void ComponentInfoFile::compile(CodeGenerator &W) const { << "static ImageInfo imageInfo {\"" << G->settings().k2_component_name.get() << "\"" << "," << std::chrono::duration_cast(now.time_since_epoch()).count() << "," << "K2_PLATFORM_HEADER_H_VERSION, " - << "{" << "}};" << NL //todo:k2 add commit hash + << "{}," //todo:k2 add commit hash + << "{}," //todo:k2 add compiler hash? + << (G->settings().k2_component_is_oneshot.get() ? "1" : "0") + << "};" << NL << "return &imageInfo;" << NL << END; W << CloseFile(); diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index b17917bab5..95f77c5735 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -224,6 +224,18 @@ void CompilerSettings::init() { } } link_file.value_ = get_full_path(link_file.get()); + if (functions_file.value_.empty()) { + if (mode.get() == "k2-component") { + functions_file.value_ = kphp_src_path.get() + "/builtin-functions/kphp-light/functions.txt"; + } else { + functions_file.value_ = kphp_src_path.get() + "/builtin-functions/kphp-full/_functions.txt"; + } + } + functions_file.value_ = get_full_path(functions_file.get()); + + if (k2_component_name.get() != "KPHP" || k2_component_is_oneshot.get()) { + kphp_error(mode.get() == "k2-component", "Options \"k2-component-name\" and \"oneshot\" available only fore k2-component mode"); + } if (mode.get() == "lib") { if (!tl_schema_file.get().empty()) { diff --git a/compiler/compiler-settings.h b/compiler/compiler-settings.h index f1cd813bb2..7410c3b66a 100644 --- a/compiler/compiler-settings.h +++ b/compiler/compiler-settings.h @@ -149,6 +149,8 @@ class CompilerSettings : vk::not_copyable { KphpOption php_code_version; KphpOption k2_component_name; + KphpOption k2_component_is_oneshot; + KphpOption cxx; KphpOption cxx_toolchain_dir; KphpOption extra_cxx_flags; diff --git a/compiler/kphp2cpp.cpp b/compiler/kphp2cpp.cpp index 5eadfe6c27..30bc28c6a9 100644 --- a/compiler/kphp2cpp.cpp +++ b/compiler/kphp2cpp.cpp @@ -212,7 +212,7 @@ int main(int argc, char *argv[]) { parser.add("Path to kphp source", settings->kphp_src_path, 's', "source-path", "KPHP_PATH", get_default_kphp_path()); parser.add("Internal file with the list of supported PHP functions", settings->functions_file, - 'f', "functions-file", "KPHP_FUNCTIONS", "${KPHP_PATH}/builtin-functions/kphp-full/_functions.txt"); + 'f', "functions-file", "KPHP_FUNCTIONS"); parser.add("File with kphp runtime sha256 hash", settings->runtime_sha256_file, "runtime-sha256", "KPHP_RUNTIME_SHA256", "${KPHP_PATH}/objs/php_lib_version.sha256"); parser.add("The output binary type: server, k2-component, cli or lib", settings->mode, @@ -295,6 +295,8 @@ int main(int argc, char *argv[]) { "require-class-typing", "KPHP_REQUIRE_CLASS_TYPING"); parser.add("Define k2 component name. Default is \"KPHP\"", settings->k2_component_name, "k2-component-name", "KPHP_K2_COMPONENT_NAME", "KPHP"); + parser.add("Enable oneshot mode to k2 component", settings->k2_component_is_oneshot, + "oneshot", "KPHP_K2_COMPONENT_IS_ONESHOT"); parser.add_implicit_option("Linker flags", settings->ld_flags); parser.add_implicit_option("Incremental linker flags", settings->incremental_linker_flags); diff --git a/runtime-light/header.h b/runtime-light/header.h index a5bfaeb645..5bae9daf93 100644 --- a/runtime-light/header.h +++ b/runtime-light/header.h @@ -17,7 +17,7 @@ #include #endif -#define K2_PLATFORM_HEADER_H_VERSION 5 +#define K2_PLATFORM_HEADER_H_VERSION 6 // Always check that enum value is a valid value! @@ -94,7 +94,7 @@ struct PlatformCtx { /* * Immediately abort component execution. - * Function is [[noreturn]] + * Function is `[[noreturn]]` */ void (*abort)(); @@ -210,7 +210,7 @@ struct PlatformCtx { * If a component has not read all updates during a `poll` iteration, the * platform is guaranteed to reschedule it. */ - char (*take_update)(uint64_t *update_d); + uint8_t (*take_update)(uint64_t *update_d); /* * Only utf-8 string supported. * Possible `level` values: @@ -241,9 +241,9 @@ enum PollStatus { PollBlocked = 0, // there is some cpu work to do; platform will reschedule component PollReschedule = 1, - // component decide to shutdown + // component decide to shutdown normally PollFinishedOk = 2, - // component decide to shutdown + // component decide to shutdown unexpectedly PollFinishedError = 3, }; @@ -255,6 +255,10 @@ struct ImageInfo { uint64_t build_timestamp; uint64_t header_h_version; uint8_t commit_hash[40]; + // TODO: more informative? + uint8_t compiler_hash[64]; + // bool + uint8_t is_oneshot; }; // Every image should provide these symbols diff --git a/runtime-light/stdlib/string-functions.cpp b/runtime-light/stdlib/string-functions.cpp index 132c36281d..055919c34f 100644 --- a/runtime-light/stdlib/string-functions.cpp +++ b/runtime-light/stdlib/string-functions.cpp @@ -34,7 +34,7 @@ void f$debug_print_string(const string &s) { Optional f$byte_to_int(const string &s) { if (s.size() != 1) { php_warning("Cannot convert non-byte string to int"); - return false; + return Optional(); } return *s.c_str(); } diff --git a/tests/k2-components/oom_loop.php b/tests/k2-components/oom_loop.php index 003a882251..7be720c1ea 100644 --- a/tests/k2-components/oom_loop.php +++ b/tests/k2-components/oom_loop.php @@ -1,7 +1,10 @@ Date: Mon, 8 Jul 2024 17:50:52 +0300 Subject: [PATCH 45/89] Add session_id function for output id --- runtime/sessions.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index ec0b8d91fe..e975756f57 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -57,7 +57,7 @@ constexpr static auto C_SECURE = "cookie_secure"; constexpr static auto C_HTTPONLY = "cookie_httponly"; // TO-DO: reconsider it -const auto skeys = vk::to_array>({ +const auto skeys = vk::to_array>({ {S_READ_CLOSE, false}, {S_DIR, string(getenv("TMPDIR")).append("sessions/")}, {S_NAME, string("PHPSESSID")}, @@ -355,7 +355,7 @@ static int session_gc(const bool &immediate = false) { continue; } else if (is_opened) { // TO-DO: fix the bug with always opened tags in the session files - // continue; + continue; } if (session_expired(path)) { @@ -511,6 +511,14 @@ bool f$session_destroy() { return true; } +Optional f$session_id() { + // if (sessions::get_sparam(sessions::S_STATUS).to_bool()) { + // php_warning("Session ID cannot be changed when a session is active"); + // return Optional{false}; + // } + return Optional{sessions::get_sparam(sessions::S_ID).to_string()}; +} + // TO-DO: implement function for changing id of existing session /* Optional f$session_id(Optional id) { From 33764f15767bb050417ddf7b03242e15148751d3 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Mon, 8 Jul 2024 17:52:08 +0300 Subject: [PATCH 46/89] Add first php test --- tests/phpt/sessions/01.php | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/phpt/sessions/01.php diff --git a/tests/phpt/sessions/01.php b/tests/phpt/sessions/01.php new file mode 100644 index 0000000000..da1a1d95b3 --- /dev/null +++ b/tests/phpt/sessions/01.php @@ -0,0 +1,51 @@ +@ok +session_id = false; + } + + function handleRequest(): ?KphpJobWorkerResponse { + $response = new Response(); + $status = session_start(); + if ($status) { + + } + $response->session_id = + } +} + +class Response implements KphpJobWorkerResponse { + /** @var AI */ + public $ai = null; + + /** @var string|false */ + public $session_id = false; +} + + +$request1 = new Request(); +$request1->ai = new A1(); +$request2 = Request(); +$request2->ai = new A2(); +$ids = array(); +kphp_job_worker_start($request, -1); + + From 718951bc6862e27c06020fcbacd79af279c65b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 10 Jul 2024 01:18:01 +0300 Subject: [PATCH 47/89] init --- .github/workflows/Build.yml | 34 +++++++++++++++++++++++++++++++--- .github/workflows/debian.yml | 11 +++++++++-- .github/workflows/ubuntu.yml | 23 ++++++++++++++++++++++- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index d3c8d31e5f..820f695dd6 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -23,23 +23,51 @@ jobs: cpp: 17 asan: off ubsan: off + light_runtime: off + - os: buster + compiler: g++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on - os: focal compiler: clang++ cpp: 17 asan: off ubsan: on + light_runtime: off + - os: focal + compiler: clang++ + cpp: 17 + asan: off + ubsan: on + light_runtime: on - os: focal compiler: g++-10 cpp: 20 asan: on ubsan: off + light_runtime: off + - os: focal + compiler: g++-10 + cpp: 20 + asan: on + ubsan: off + light_runtime: on - os: jammy compiler: g++ cpp: 20 asan: on ubsan: off - - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" + light_runtime: off + - os: jammy + compiler: g++ + cpp: 20 + asan: on + ubsan: off + light_runtime: on + + name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" steps: - uses: actions/checkout@v3 @@ -78,7 +106,7 @@ jobs: - name: Build all run: docker exec kphp-build-container-${{matrix.os}} bash -c - "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" + "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DCOMPILE_RUNTIME_LIGHT=${{matrix.light_runtime}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" - name: Run unit tests run: docker exec kphp-build-container-${{matrix.os}} bash -c diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 2d507d8110..1d365bd178 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -21,8 +21,15 @@ jobs: cpp: 17 asan: off ubsan: off + light_runtime: off + - os: buster + compiler: g++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" + name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" steps: - uses: actions/checkout@v3 @@ -59,7 +66,7 @@ jobs: - name: Build all run: docker exec -u kitten kphp-build-container-${{matrix.os}} bash -c - "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" + "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DCOMPILE_RUNTIME_LIGHT=${{matrix.light_runtime}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" - name: Run unit tests run: docker exec -u kitten kphp-build-container-${{matrix.os}} bash -c diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 49af16bd9f..0280b01a48 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -21,18 +21,39 @@ jobs: cpp: 17 asan: off ubsan: off + light_runtime: off + - os: jammy + compiler: g++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on - os: focal compiler: clang++ cpp: 17 asan: off ubsan: on + light_runtime: off + - os: focal + compiler: clang++ + cpp: 17 + asan: off + ubsan: on + light_runtime: on + - os: focal + compiler: g++-10 + cpp: 20 + asan: on + ubsan: off + light_runtime: off - os: focal compiler: g++-10 cpp: 20 asan: on ubsan: off + light_runtime: on - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" + name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" steps: - uses: actions/checkout@v3 From 5db00ea3fdaf2ec29108e3048eb97c9f0e08f150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 10 Jul 2024 01:25:29 +0300 Subject: [PATCH 48/89] temporaly disable buster build die to sury issue --- .github/workflows/Build.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 820f695dd6..65b52c17b1 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -18,18 +18,18 @@ jobs: strategy: matrix: include: - - os: buster - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: off - - os: buster - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: on + # - os: buster + # compiler: g++ + # cpp: 17 + # asan: off + # ubsan: off + # light_runtime: off + # - os: buster + # compiler: g++ + # cpp: 17 + # asan: off + # ubsan: off + # light_runtime: on - os: focal compiler: clang++ cpp: 17 From a2d8afb232afab8afb626ea69b0a1690112248fe Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Wed, 10 Jul 2024 10:21:01 +0300 Subject: [PATCH 49/89] Add session_id(), hide session_destroy() --- runtime/sessions.cpp | 45 ++++++++++++++++++-------------------------- runtime/sessions.h | 6 +++--- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index e975756f57..ffa31dcc4c 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -72,7 +72,6 @@ const auto skeys = vk::to_array>({ }); static void initialize_sparams(const array &options) noexcept { - reset_sparams(); for (const auto& it : skeys) { if (options.isset(string(it.first))) { set_sparam(it.first, options.get_value(string(it.first))); @@ -316,7 +315,7 @@ static bool session_expired(const string &path) { if (ret_ctime < 0 or ret_lifetime < 0) { php_warning("Failed to get metadata of the file on path: %s", path.c_str()); return false; - } + } int now = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); if (ctime + lifetime <= now) { @@ -414,6 +413,8 @@ static bool session_start() { } } } + } else if (strpbrk(get_sparam(S_ID).to_string().c_str(), "\r\n\t <>'\"\\")) { + set_sparam(S_ID, false); } if (!session_initialize()) { @@ -501,35 +502,25 @@ array f$session_get_cookie_params() { return sessions::session_get_cookie_params(); } -// TO-DO: is this correct? -bool f$session_destroy() { - if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { - php_warning("Trying to destroy uninitialized session"); - return false; - } - sessions::session_close(); - return true; -} - -Optional f$session_id() { - // if (sessions::get_sparam(sessions::S_STATUS).to_bool()) { - // php_warning("Session ID cannot be changed when a session is active"); - // return Optional{false}; - // } - return Optional{sessions::get_sparam(sessions::S_ID).to_string()}; -} - -// TO-DO: implement function for changing id of existing session -/* -Optional f$session_id(Optional id) { +Optional f$session_id(const Optional &id) { if (id.has_value() && sessions::get_sparam(sessions::S_STATUS).to_bool()) { php_warning("Session ID cannot be changed when a session is active"); return Optional{false}; } - // TO-DO: check headers_sent() + + mixed prev_id = sessions::get_sparam(sessions::S_ID); if (id.has_value()) { - sessions::set_sparam(sessions::S_ID, string(id)); + sessions::set_sparam(sessions::S_ID, id.val()); } - return Optional{sessions::get_sparam(sessions::S_ID).to_string()}; + return (prev_id.is_bool()) ? Optional{false} : Optional(prev_id.as_string()); } -*/ \ No newline at end of file + +// TO-DO +// bool f$session_destroy() { +// if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { +// php_warning("Trying to destroy uninitialized session"); +// return false; +// } +// sessions::session_close(); +// return true; +// } diff --git a/runtime/sessions.h b/runtime/sessions.h index 2de3ae7c14..caf9f6b70b 100644 --- a/runtime/sessions.h +++ b/runtime/sessions.h @@ -12,7 +12,7 @@ int64_t f$session_status(); Optional f$session_encode(); bool f$session_decode(const string &data); array f$session_get_cookie_params(); +Optional f$session_id(const Optional &id = Optional()); -// TO-DO: -// bool f$session_destroy(); -// Optional f$session_id(Optional id = Optional()); \ No newline at end of file +// TO-DO +// bool f$session_destroy(); \ No newline at end of file From 954616e5f45052ab65225db7557693c058244fe7 Mon Sep 17 00:00:00 2001 From: Alexander Polyakov Date: Wed, 10 Jul 2024 17:40:30 +0300 Subject: [PATCH 50/89] Make RPC in light runtime more robust (#1037) Fix a bug in store_string caused by incorrect usage of string_buffer. Add RpcBuffer class that prevents us from similar bugs in the future. --- .gitignore | 1 + runtime-light/stdlib/rpc/rpc-api.cpp | 160 ++++++++---------- runtime-light/stdlib/rpc/rpc-api.h | 3 +- runtime-light/stdlib/rpc/rpc-buffer.h | 78 +++++++++ runtime-light/stdlib/rpc/rpc-context.h | 32 +--- .../stdlib/rpc/rpc-extra-headers.cpp | 2 +- runtime-light/stdlib/rpc/rpc-extra-info.cpp | 2 + runtime-light/stdlib/rpc/rpc-extra-info.h | 2 - runtime-light/stdlib/rpc/rpc-tl-defs.h | 2 +- runtime-light/stdlib/rpc/rpc-tl-query.cpp | 2 +- runtime-light/stdlib/rpc/rpc-tl-request.cpp | 2 +- runtime-light/tl/tl-builtins.cpp | 8 +- runtime-light/tl/tl-builtins.h | 5 +- tests/k2-components/scheme.tl | 12 ++ tests/k2-components/test_rpc_memcached.php | 36 ++++ 15 files changed, 212 insertions(+), 135 deletions(-) create mode 100644 runtime-light/stdlib/rpc/rpc-buffer.h create mode 100644 tests/k2-components/scheme.tl create mode 100644 tests/k2-components/test_rpc_memcached.php diff --git a/.gitignore b/.gitignore index 3df1ffe88a..6437597584 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ .DS_Store .idea +.vscode build *.iml /cmake-build-* diff --git a/runtime-light/stdlib/rpc/rpc-api.cpp b/runtime-light/stdlib/rpc/rpc-api.cpp index 0e676225ee..48e3bf9ed4 100644 --- a/runtime-light/stdlib/rpc/rpc-api.cpp +++ b/runtime-light/stdlib/rpc/rpc-api.cpp @@ -6,19 +6,19 @@ #include #include -#include #include +#include #include #include #include "common/algorithms/find.h" #include "common/rpc-error-codes.h" -#include "common/tl/constants/common.h" +#include "runtime-core/utils/kphp-assert-core.h" #include "runtime-light/allocator/allocator.h" +#include "runtime-light/stdlib/rpc/rpc-buffer.h" #include "runtime-light/stdlib/rpc/rpc-context.h" #include "runtime-light/stdlib/rpc/rpc-extra-headers.h" #include "runtime-light/streams/interface.h" -#include "runtime-light/utils/concepts.h" namespace rpc_impl_ { @@ -46,10 +46,6 @@ mixed mixed_array_get_value(const mixed &arr, const string &str_key, int64_t num return {}; } -bool rpc_fetch_remaining_enough(size_t len) noexcept { - return RpcComponentContext::get().fetch_state.remaining() >= len; -} - array make_fetch_error(string &&error_msg, int32_t error_code) { array res; res.set_value(string{"__error", 7}, std::move(error_msg)); @@ -57,18 +53,6 @@ array make_fetch_error(string &&error_msg, int32_t error_code) { return res; } -template -std::optional fetch_trivial() noexcept { - if (!rpc_fetch_remaining_enough(sizeof(T))) { - return {}; // TODO: error handling - } - - auto &rpc_ctx{RpcComponentContext::get()}; - const auto v{*reinterpret_cast(rpc_ctx.buffer.c_str() + rpc_ctx.fetch_state.pos())}; - rpc_ctx.fetch_state.adjust(sizeof(T)); - return v; -} - array fetch_function_untyped(const class_instance &rpc_query) noexcept { php_assert(!rpc_query.is_null()); CurrentTlQuery::get().set_current_tl_function(rpc_query); @@ -94,12 +78,6 @@ class_instance fetch_function_typed(const class_instance -bool store_trivial(T v) noexcept { - RpcComponentContext::get().buffer.append(reinterpret_cast(&v), sizeof(T)); - return true; -} - class_instance store_function(const mixed &tl_object) noexcept { auto &cur_query{CurrentTlQuery::get()}; const auto &rpc_image_state{RpcImageState::get()}; @@ -120,27 +98,26 @@ class_instance store_function(const mixed &tl_object) noexcept { return rpc_tl_query; } -task_t rpc_send_impl(const string &actor, double timeout, bool ignore_answer) noexcept { - auto &rpc_ctx = RpcComponentContext::get(); - +task_t rpc_send_impl(string actor, double timeout, bool ignore_answer) noexcept { if (timeout <= 0 || timeout > MAX_TIMEOUT_S) { // TODO: handle timeouts // timeout = conn.get()->timeout_ms * 0.001; } + auto &rpc_ctx = RpcComponentContext::get(); string request_buf{}; - size_t request_size{rpc_ctx.buffer.size()}; + size_t request_size{rpc_ctx.rpc_buffer.size()}; // 'request_buf' will look like this: // [ RpcExtraHeaders (optional) ] [ payload ] - if (const auto [opt_new_extra_header, cur_extra_header_size]{regularize_extra_headers(rpc_ctx.buffer.c_str(), ignore_answer)}; opt_new_extra_header) { + if (const auto [opt_new_extra_header, cur_extra_header_size]{regularize_extra_headers(rpc_ctx.rpc_buffer.data(), ignore_answer)}; opt_new_extra_header) { const auto new_extra_header{opt_new_extra_header.value()}; const auto new_extra_header_size{sizeof(std::decay_t)}; request_size = request_size - cur_extra_header_size + new_extra_header_size; request_buf.append(reinterpret_cast(&new_extra_header), new_extra_header_size); - request_buf.append(rpc_ctx.buffer.c_str() + cur_extra_header_size, rpc_ctx.buffer.size() - cur_extra_header_size); + request_buf.append(rpc_ctx.rpc_buffer.data() + cur_extra_header_size, rpc_ctx.rpc_buffer.size() - cur_extra_header_size); } else { - request_buf.append(rpc_ctx.buffer.c_str(), request_size); + request_buf.append(rpc_ctx.rpc_buffer.data(), request_size); } // get timestamp before co_await to also count the time we were waiting for runtime to resume this coroutine @@ -160,7 +137,7 @@ task_t rpc_send_impl(const string &actor, double timeout, bool ign co_return RpcQueryInfo{.id = query_id, .request_size = request_size, .timestamp = timestamp}; } -task_t rpc_tl_query_one_impl(const string &actor, mixed tl_object, double timeout, bool collect_resp_extra_info, bool ignore_answer) noexcept { +task_t rpc_tl_query_one_impl(string actor, mixed tl_object, double timeout, bool collect_resp_extra_info, bool ignore_answer) noexcept { auto &rpc_ctx{RpcComponentContext::get()}; if (!tl_object.is_array()) { @@ -168,7 +145,7 @@ task_t rpc_tl_query_one_impl(const string &actor, mixed tl_object, co_return RpcQueryInfo{}; } - rpc_ctx.buffer.clean(); + rpc_ctx.rpc_buffer.clean(); auto rpc_tl_query{store_function(tl_object)}; // TODO: exception handling if (rpc_tl_query.is_null()) { co_return RpcQueryInfo{}; @@ -186,7 +163,7 @@ task_t rpc_tl_query_one_impl(const string &actor, mixed tl_object, co_return query_info; } -task_t typed_rpc_tl_query_one_impl(const string &actor, const RpcRequest &rpc_request, double timeout, bool collect_responses_extra_info, +task_t typed_rpc_tl_query_one_impl(string actor, const RpcRequest &rpc_request, double timeout, bool collect_responses_extra_info, bool ignore_answer) noexcept { auto &rpc_ctx{RpcComponentContext::get()}; @@ -195,7 +172,7 @@ task_t typed_rpc_tl_query_one_impl(const string &actor, const RpcR co_return RpcQueryInfo{}; } - rpc_ctx.buffer.clean(); + rpc_ctx.rpc_buffer.clean(); auto fetcher{rpc_request.store_request()}; if (!static_cast(fetcher)) { rpc_ctx.current_query.raise_storing_error("could not store rpc request"); @@ -263,9 +240,8 @@ task_t> rpc_tl_query_result_one_impl(int64_t query_id) noexcept { it_response_extra_info->second.first = rpc_response_extra_info_status_t::READY; } - rpc_ctx.fetch_state.reset(0, data.size()); - rpc_ctx.buffer.clean(); - rpc_ctx.buffer.append(data.c_str(), data.size()); + rpc_ctx.rpc_buffer.clean(); + rpc_ctx.rpc_buffer.store(data.c_str(), data.size()); co_return fetch_function_untyped(rpc_query); } @@ -315,9 +291,8 @@ task_t> typed_rpc_tl_query_result_one_impl(i it_response_extra_info->second.first = rpc_response_extra_info_status_t::READY; } - rpc_ctx.fetch_state.reset(0, data.size()); - rpc_ctx.buffer.clean(); - rpc_ctx.buffer.append(data.c_str(), data.size()); + rpc_ctx.rpc_buffer.clean(); + rpc_ctx.rpc_buffer.store(data.c_str(), data.size()); co_return fetch_function_typed(rpc_query, error_factory); } @@ -330,78 +305,84 @@ bool f$store_int(int64_t v) noexcept { if (unlikely(is_int32_overflow(v))) { php_warning("Got int32 overflow on storing '%" PRIi64 "', the value will be casted to '%d'", v, static_cast(v)); } - return rpc_impl_::store_trivial(static_cast(v)); + RpcComponentContext::get().rpc_buffer.store_trivial(static_cast(v)); + return true; } bool f$store_long(int64_t v) noexcept { - return rpc_impl_::store_trivial(v); + RpcComponentContext::get().rpc_buffer.store_trivial(v); + return true; } bool f$store_float(double v) noexcept { - return rpc_impl_::store_trivial(static_cast(v)); + RpcComponentContext::get().rpc_buffer.store_trivial(static_cast(v)); + return true; } bool f$store_double(double v) noexcept { - return rpc_impl_::store_trivial(v); + RpcComponentContext::get().rpc_buffer.store_trivial(v); + return true; } bool f$store_string(const string &v) noexcept { // TODO: support large strings - auto &buffer{RpcComponentContext::get().buffer}; + auto &rpc_buf{RpcComponentContext::get().rpc_buffer}; string::size_type string_len{v.size()}; string::size_type size_len{}; if (string_len <= rpc_impl_::SMALL_STRING_MAX_LEN) { - buffer << static_cast(string_len); size_len = 1; + rpc_buf.store_trivial(static_cast(string_len)); } else if (string_len <= rpc_impl_::MEDIUM_STRING_MAX_LEN) { - buffer << static_cast(rpc_impl_::MEDIUM_STRING_MAGIC) << static_cast(string_len & 0xff) << static_cast((string_len >> 8) & 0xff) - << static_cast((string_len >> 16) & 0xff); size_len = 4; + rpc_buf.store_trivial(static_cast(rpc_impl_::MEDIUM_STRING_MAGIC)); + rpc_buf.store_trivial(static_cast(string_len & 0xff)); + rpc_buf.store_trivial(static_cast((string_len >> 8) & 0xff)); + rpc_buf.store_trivial(static_cast((string_len >> 16) & 0xff)); } else { - buffer << static_cast(rpc_impl_::LARGE_STRING_MAGIC) << static_cast(string_len & 0xff) << static_cast((string_len >> 8) & 0xff) - << static_cast((string_len >> 16) & 0xff) << static_cast((string_len >> 24) & 0xff); -// buffer << static_cast(rpc_impl_::LARGE_STRING_MAGIC) << static_cast(string_len & 0xff) << static_cast((string_len >> 8) & 0xff) -// << static_cast((string_len >> 16) & 0xff) << static_cast((string_len >> 24) & 0xff) -// << static_cast((string_len >> 32) & 0xff) << static_cast((string_len >> 40) & 0xff) -// << static_cast((string_len >> 48) & 0xff); size_len = 8; + rpc_buf.store_trivial(static_cast(rpc_impl_::LARGE_STRING_MAGIC)); + rpc_buf.store_trivial(static_cast(string_len & 0xff)); + rpc_buf.store_trivial(static_cast((string_len >> 8) & 0xff)); + rpc_buf.store_trivial(static_cast((string_len >> 16) & 0xff)); + rpc_buf.store_trivial(static_cast((string_len >> 24) & 0xff)); + // rpc_buf.store_trivial(static_cast((string_len >> 32) & 0xff)); + // rpc_buf.store_trivial(static_cast((string_len >> 40) & 0xff)); + // rpc_buf.store_trivial(static_cast((string_len >> 48) & 0xff)); } - buffer.append(v.c_str(), static_cast(string_len)); + rpc_buf.store(v.c_str(), string_len); const auto total_len{size_len + string_len}; const auto total_len_with_padding{(total_len + 3) & ~static_cast(3)}; const auto padding{total_len_with_padding - total_len}; std::array padding_array{'\0', '\0', '\0', '\0'}; - buffer.append(padding_array.data(), padding); + rpc_buf.store(padding_array.data(), padding); return true; } // === Rpc Fetch ================================================================================== int64_t f$fetch_int() noexcept { - const auto opt{rpc_impl_::fetch_trivial()}; - return opt.value_or(0); + return static_cast(RpcComponentContext::get().rpc_buffer.fetch_trivial().value_or(0)); } int64_t f$fetch_long() noexcept { - const auto opt{rpc_impl_::fetch_trivial()}; - return opt.value_or(0); + return RpcComponentContext::get().rpc_buffer.fetch_trivial().value_or(0); } double f$fetch_double() noexcept { - const auto opt{rpc_impl_::fetch_trivial()}; - return opt.value_or(0.0); + return RpcComponentContext::get().rpc_buffer.fetch_trivial().value_or(0.0); } double f$fetch_float() noexcept { - const auto opt{rpc_impl_::fetch_trivial()}; - return static_cast(opt.value_or(0.0)); + return static_cast(RpcComponentContext::get().rpc_buffer.fetch_trivial().value_or(0.0)); } string f$fetch_string() noexcept { + auto &rpc_buf{RpcComponentContext::get().rpc_buffer}; + uint8_t first_byte{}; - if (const auto opt_first_byte{rpc_impl_::fetch_trivial()}; opt_first_byte) { + if (const auto opt_first_byte{rpc_buf.fetch_trivial()}; opt_first_byte) { first_byte = opt_first_byte.value(); } else { return {}; // TODO: error handling @@ -412,16 +393,16 @@ string f$fetch_string() noexcept { switch (first_byte) { case rpc_impl_::LARGE_STRING_MAGIC: { // next 7 bytes are string's length // TODO: support large strings // static_assert(sizeof(string::size_type) >= 8, "string's length doesn't fit platform size"); - if (!rpc_impl_::rpc_fetch_remaining_enough(7)) { + if (rpc_buf.remaining() < 7) { return {}; // TODO: error handling } - const auto first{static_cast(rpc_impl_::fetch_trivial().value())}; - const auto second{static_cast(rpc_impl_::fetch_trivial().value()) << 8}; - const auto third{static_cast(rpc_impl_::fetch_trivial().value()) << 16}; - const auto fourth{static_cast(rpc_impl_::fetch_trivial().value()) << 24}; - const auto fifth{static_cast(rpc_impl_::fetch_trivial().value()) << 32}; - const auto sixth{static_cast(rpc_impl_::fetch_trivial().value()) << 40}; - const auto seventh{static_cast(rpc_impl_::fetch_trivial().value()) << 48}; + const auto first{static_cast(rpc_buf.fetch_trivial().value())}; + const auto second{static_cast(rpc_buf.fetch_trivial().value()) << 8}; + const auto third{static_cast(rpc_buf.fetch_trivial().value()) << 16}; + const auto fourth{static_cast(rpc_buf.fetch_trivial().value()) << 24}; + const auto fifth{static_cast(rpc_buf.fetch_trivial().value()) << 32}; + const auto sixth{static_cast(rpc_buf.fetch_trivial().value()) << 40}; + const auto seventh{static_cast(rpc_buf.fetch_trivial().value()) << 48}; string_len = first | second | third | fourth | fifth | sixth | seventh; if (string_len < (1 << 24)) { php_warning("long string's length is less than 1 << 24"); @@ -429,12 +410,12 @@ string f$fetch_string() noexcept { size_len = 8; } case rpc_impl_::MEDIUM_STRING_MAGIC: { // next 3 bytes are string's length - if (!rpc_impl_::rpc_fetch_remaining_enough(3)) { + if (rpc_buf.remaining() < 3) { return {}; // TODO: error handling } - const auto first{static_cast(rpc_impl_::fetch_trivial().value())}; - const auto second{static_cast(rpc_impl_::fetch_trivial().value()) << 8}; - const auto third{static_cast(rpc_impl_::fetch_trivial().value()) << 16}; + const auto first{static_cast(rpc_buf.fetch_trivial().value())}; + const auto second{static_cast(rpc_buf.fetch_trivial().value()) << 8}; + const auto third{static_cast(rpc_buf.fetch_trivial().value()) << 16}; string_len = first | second | third; if (string_len <= 253) { php_warning("long string's length is less than 254"); @@ -447,13 +428,12 @@ string f$fetch_string() noexcept { } const auto total_len_with_padding{(size_len + string_len + 3) & ~static_cast(3)}; - if (!rpc_impl_::rpc_fetch_remaining_enough(total_len_with_padding - size_len)) { + if (rpc_buf.remaining() < total_len_with_padding - size_len) { return {}; // TODO: error handling } - auto &rpc_ctx{RpcComponentContext::get()}; - string res{rpc_ctx.buffer.c_str() + rpc_ctx.fetch_state.pos(), string_len}; - rpc_ctx.fetch_state.adjust(total_len_with_padding - size_len); + string res{rpc_buf.data() + rpc_buf.pos(), string_len}; + rpc_buf.adjust(total_len_with_padding - size_len); return res; } @@ -492,9 +472,7 @@ task_t>> f$rpc_tl_query_result(array query_ids) noex // === Rpc Misc ================================================================================== void f$rpc_clean() noexcept { - auto &rpc_ctx{RpcComponentContext::get()}; - rpc_ctx.buffer.clean(); - rpc_ctx.fetch_state.reset(0, 0); + RpcComponentContext::get().rpc_buffer.clean(); } // === Misc ======================================================================================= @@ -508,15 +486,15 @@ bool is_int32_overflow(int64_t v) noexcept { } void store_raw_vector_double(const array &vector) noexcept { // TODO: didn't we forget vector's length? - RpcComponentContext::get().buffer.append(reinterpret_cast(vector.get_const_vector_pointer()), sizeof(double) * vector.count()); + RpcComponentContext::get().rpc_buffer.store(reinterpret_cast(vector.get_const_vector_pointer()), sizeof(double) * vector.count()); } void fetch_raw_vector_double(array &vector, int64_t num_elems) noexcept { + auto &rpc_buf{RpcComponentContext::get().rpc_buffer}; const auto len_bytes{sizeof(double) * num_elems}; - if (!rpc_impl_::rpc_fetch_remaining_enough(len_bytes)) { + if (rpc_buf.remaining() < len_bytes) { return; // TODO: error handling } - auto &rpc_ctx{RpcComponentContext::get()}; - vector.memcpy_vector(num_elems, rpc_ctx.buffer.c_str() + rpc_ctx.fetch_state.pos()); - rpc_ctx.fetch_state.adjust(len_bytes); + vector.memcpy_vector(num_elems, rpc_buf.data() + rpc_buf.pos()); + rpc_buf.adjust(len_bytes); } diff --git a/runtime-light/stdlib/rpc/rpc-api.h b/runtime-light/stdlib/rpc/rpc-api.h index 2bf19203af..50522a3e92 100644 --- a/runtime-light/stdlib/rpc/rpc-api.h +++ b/runtime-light/stdlib/rpc/rpc-api.h @@ -6,7 +6,6 @@ #include #include -#include #include "runtime-core/runtime-core.h" #include "runtime-light/coroutine/task.h" @@ -27,7 +26,7 @@ struct RpcQueryInfo { double timestamp{0.0}; }; -task_t typed_rpc_tl_query_one_impl(const string &actor, const RpcRequest &rpc_request, double timeout, bool collect_responses_extra_info, +task_t typed_rpc_tl_query_one_impl(string actor, const RpcRequest &rpc_request, double timeout, bool collect_responses_extra_info, bool ignore_answer) noexcept; task_t> typed_rpc_tl_query_result_one_impl(int64_t query_id, const RpcErrorFactory &error_factory) noexcept; diff --git a/runtime-light/stdlib/rpc/rpc-buffer.h b/runtime-light/stdlib/rpc/rpc-buffer.h new file mode 100644 index 0000000000..3d8f0b0f3e --- /dev/null +++ b/runtime-light/stdlib/rpc/rpc-buffer.h @@ -0,0 +1,78 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "common/mixin/not_copyable.h" +#include "runtime-core/runtime-core.h" +#include "runtime-core/utils/kphp-assert-core.h" +#include "runtime-light/utils/concepts.h" + +class RpcBuffer : private vk::not_copyable { + string_buffer m_buffer; + size_t m_pos{0}; + size_t m_remaining{0}; + +public: + RpcBuffer() = default; + + const char *data() const noexcept { + return m_buffer.buffer(); + } + + size_t size() const noexcept { + return static_cast(m_buffer.size()); + } + + size_t remaining() const noexcept { + return m_remaining; + } + + size_t pos() const noexcept { + return m_pos; + } + + void clean() noexcept { + m_buffer.clean(); + m_pos = 0; + m_remaining = 0; + } + + void reset(size_t pos) noexcept { + php_assert(pos >= 0 && pos <= size()); + m_pos = pos; + m_remaining = size() - m_pos; + } + + void adjust(size_t len) noexcept { + php_assert(m_pos + len <= size()); + m_pos += len; + m_remaining -= len; + } + + void store(const char *src, size_t len) noexcept { + m_buffer.append(src, len); + m_remaining += len; + } + + template + void store_trivial(const T &t) noexcept { + store(reinterpret_cast(&t), sizeof(T)); + } + + template + std::optional fetch_trivial() noexcept { + if (m_remaining < sizeof(T)) { + return std::nullopt; + } + + const auto t{*reinterpret_cast(m_buffer.c_str() + m_pos)}; + m_pos += sizeof(T); + m_remaining -= sizeof(T); + return t; + } +}; diff --git a/runtime-light/stdlib/rpc/rpc-context.h b/runtime-light/stdlib/rpc/rpc-context.h index f802246854..8a4a0ca203 100644 --- a/runtime-light/stdlib/rpc/rpc-context.h +++ b/runtime-light/stdlib/rpc/rpc-context.h @@ -10,43 +10,17 @@ #include "common/mixin/not_copyable.h" #include "runtime-core/memory-resource/resource_allocator.h" #include "runtime-core/runtime-core.h" +#include "runtime-light/stdlib/rpc/rpc-buffer.h" #include "runtime-light/stdlib/rpc/rpc-extra-info.h" #include "runtime-light/stdlib/rpc/rpc-tl-defs.h" #include "runtime-light/stdlib/rpc/rpc-tl-query.h" #include "runtime-light/streams/component-stream.h" struct RpcComponentContext final : private vk::not_copyable { - class FetchState { - size_t m_pos{0}; - size_t m_remaining{0}; - - public: - constexpr FetchState() = default; - - constexpr size_t remaining() const noexcept { - return m_remaining; - } - - constexpr size_t pos() const noexcept { - return m_pos; - } - - constexpr void reset(size_t pos, size_t len) noexcept { - m_pos = pos; - m_remaining = len; - } - - constexpr void adjust(size_t len) noexcept { - m_pos += len; - m_remaining -= len; - } - }; - template using unordered_map = memory_resource::stl::unordered_map; - string_buffer buffer; - FetchState fetch_state; + RpcBuffer rpc_buffer; int64_t current_query_id{0}; CurrentTlQuery current_query; unordered_map> pending_component_queries; @@ -58,6 +32,8 @@ struct RpcComponentContext final : private vk::not_copyable { static RpcComponentContext &get() noexcept; }; +// ================================================================================================ + struct RpcImageState final : private vk::not_copyable { array tl_storers_ht; tl_fetch_wrapper_ptr tl_fetch_wrapper{nullptr}; diff --git a/runtime-light/stdlib/rpc/rpc-extra-headers.cpp b/runtime-light/stdlib/rpc/rpc-extra-headers.cpp index 60cf80e946..b50c9334b0 100644 --- a/runtime-light/stdlib/rpc/rpc-extra-headers.cpp +++ b/runtime-light/stdlib/rpc/rpc-extra-headers.cpp @@ -8,7 +8,7 @@ #include "common/algorithms/find.h" #include "common/tl/constants/common.h" -#include "runtime-light/utils/php_assert.h" +#include "runtime-core/utils/kphp-assert-core.h" namespace { diff --git a/runtime-light/stdlib/rpc/rpc-extra-info.cpp b/runtime-light/stdlib/rpc/rpc-extra-info.cpp index f30401c16f..70779af0ec 100644 --- a/runtime-light/stdlib/rpc/rpc-extra-info.cpp +++ b/runtime-light/stdlib/rpc/rpc-extra-info.cpp @@ -4,6 +4,8 @@ #include "runtime-light/stdlib/rpc/rpc-extra-info.h" +#include "common/algorithms/hashes.h" +#include "common/wrappers/string_view.h" #include "runtime-light/stdlib/rpc/rpc-context.h" const char *C$KphpRpcRequestsExtraInfo::get_class() const noexcept { diff --git a/runtime-light/stdlib/rpc/rpc-extra-info.h b/runtime-light/stdlib/rpc/rpc-extra-info.h index 7f91818f26..32fa5e3527 100644 --- a/runtime-light/stdlib/rpc/rpc-extra-info.h +++ b/runtime-light/stdlib/rpc/rpc-extra-info.h @@ -7,8 +7,6 @@ #include #include -#include "common/algorithms/hashes.h" -#include "common/wrappers/string_view.h" #include "runtime-core/class-instance/refcountable-php-classes.h" #include "runtime-core/runtime-core.h" diff --git a/runtime-light/stdlib/rpc/rpc-tl-defs.h b/runtime-light/stdlib/rpc/rpc-tl-defs.h index 9ad4258f93..b130d5e01f 100644 --- a/runtime-light/stdlib/rpc/rpc-tl-defs.h +++ b/runtime-light/stdlib/rpc/rpc-tl-defs.h @@ -7,9 +7,9 @@ #include #include "runtime-core/runtime-core.h" +#include "runtime-core/utils/kphp-assert-core.h" #include "runtime-light/stdlib/rpc/rpc-tl-func-base.h" #include "runtime-light/stdlib/rpc/rpc-tl-function.h" -#include "runtime-light/utils/php_assert.h" using tl_undefined_php_type = std::nullptr_t; using tl_storer_ptr = std::unique_ptr (*)(const mixed &); diff --git a/runtime-light/stdlib/rpc/rpc-tl-query.cpp b/runtime-light/stdlib/rpc/rpc-tl-query.cpp index 97b709db99..e0bfc90902 100644 --- a/runtime-light/stdlib/rpc/rpc-tl-query.cpp +++ b/runtime-light/stdlib/rpc/rpc-tl-query.cpp @@ -6,8 +6,8 @@ #include +#include "runtime-core/utils/kphp-assert-core.h" #include "runtime-light/stdlib/rpc/rpc-context.h" -#include "runtime-light/utils/php_assert.h" void CurrentTlQuery::reset() noexcept { current_tl_function_name = string{}; diff --git a/runtime-light/stdlib/rpc/rpc-tl-request.cpp b/runtime-light/stdlib/rpc/rpc-tl-request.cpp index b46274d7ae..aa41e5a589 100644 --- a/runtime-light/stdlib/rpc/rpc-tl-request.cpp +++ b/runtime-light/stdlib/rpc/rpc-tl-request.cpp @@ -4,7 +4,7 @@ #include "runtime-light/stdlib/rpc/rpc-tl-request.h" -#include "runtime-light/utils/php_assert.h" +#include "runtime-core/utils/kphp-assert-core.h" RpcRequestResult::RpcRequestResult(bool is_typed, std::unique_ptr &&result_fetcher) : is_typed(is_typed) diff --git a/runtime-light/tl/tl-builtins.cpp b/runtime-light/tl/tl-builtins.cpp index e10de69d41..0f1c236a06 100644 --- a/runtime-light/tl/tl-builtins.cpp +++ b/runtime-light/tl/tl-builtins.cpp @@ -11,15 +11,15 @@ void register_tl_storers_table_and_fetcher(const array &gen$ht, t } int32_t tl_parse_save_pos() { - return static_cast(RpcComponentContext::get().fetch_state.pos()); + return static_cast(RpcComponentContext::get().rpc_buffer.pos()); } bool tl_parse_restore_pos(int32_t pos) { - auto &rpc_ctx{RpcComponentContext::get()}; - if (pos < 0 || pos > rpc_ctx.fetch_state.pos()) { + auto &rpc_buf{RpcComponentContext::get().rpc_buffer}; + if (pos < 0 || pos > rpc_buf.pos()) { return false; } - rpc_ctx.fetch_state.reset(static_cast(pos), rpc_ctx.fetch_state.remaining() + rpc_ctx.fetch_state.pos() - pos); + rpc_buf.reset(pos); return true; } diff --git a/runtime-light/tl/tl-builtins.h b/runtime-light/tl/tl-builtins.h index 91b5b325d4..1b524fe2b5 100644 --- a/runtime-light/tl/tl-builtins.h +++ b/runtime-light/tl/tl-builtins.h @@ -10,12 +10,9 @@ #include "common/php-functions.h" #include "common/tl/constants/common.h" #include "runtime-core/runtime-core.h" +#include "runtime-core/utils/kphp-assert-core.h" #include "runtime-light/stdlib/rpc/rpc-api.h" -#include "runtime-light/stdlib/rpc/rpc-context.h" #include "runtime-light/stdlib/rpc/rpc-tl-defs.h" -#include "runtime-light/stdlib/rpc/rpc-tl-function.h" -#include "runtime-light/stdlib/rpc/rpc-tl-kphp-request.h" -#include "runtime-light/utils/php_assert.h" // TODO: get rid of it here #define CHECK_EXCEPTION(action) diff --git a/tests/k2-components/scheme.tl b/tests/k2-components/scheme.tl new file mode 100644 index 0000000000..c4d7c8ca07 --- /dev/null +++ b/tests/k2-components/scheme.tl @@ -0,0 +1,12 @@ +// MemCache + +memcache.not_found#32c42422 = memcache.Value; +memcache.strvalue#a6bebb1a value:string flags:int = memcache.Value; + +---functions--- + +// MemCache + +@read memcache.get#d33b13ae key:string = memcache.Value; + +@write memcache.set#eeeb54c4 key:string flags:int delay:int value:string = Bool; diff --git a/tests/k2-components/test_rpc_memcached.php b/tests/k2-components/test_rpc_memcached.php new file mode 100644 index 0000000000..7f1a907129 --- /dev/null +++ b/tests/k2-components/test_rpc_memcached.php @@ -0,0 +1,36 @@ + "memcache.get", "key" => "xxx"]]); + $response = rpc_tl_query_result($ids)[0]; + $result = $response["result"]; + if ($result["_"] !== "memcache.not_found") { + warning("memcache.not_found expected"); + return false; + } + + $ids = rpc_tl_query("mc_main", [["_" => "memcache.set", "key" => "foo", "flags" => 0, "delay" => 0, "value" => "bar"]]); + $response = rpc_tl_query_result($ids)[0]; + $result = $response["result"]; + if ($result !== true) { + warning("true expected"); + return false; + } + + $ids = rpc_tl_query("mc_main", [["_" => "memcache.get", "key" => "foo"]]); + $response = rpc_tl_query_result($ids)[0]; + $result = $response["result"]; + if ($result["_"] !== "memcache.strvalue" || $result["value"] !== "bar") { + warning("\"bar\" expected"); + return false; + } + + return true; +} + +component_server_get_query(); +if (do_untyped_rpc()) { + component_server_send_result("OK"); +} else { + component_server_send_result("FAIL"); +} From c42a920454f8271a85886e5716d0cffea728f113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 10 Jul 2024 20:52:04 +0300 Subject: [PATCH 51/89] update --- .github/workflows/Build.yml | 44 +++++++++++------------------ .github/workflows/Dockerfile.buster | 8 +++--- .github/workflows/debian.yml | 6 ++++ .github/workflows/ubuntu.yml | 23 +-------------- 4 files changed, 27 insertions(+), 54 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 65b52c17b1..14c8941479 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -18,54 +18,42 @@ jobs: strategy: matrix: include: - # - os: buster - # compiler: g++ - # cpp: 17 - # asan: off - # ubsan: off - # light_runtime: off - # - os: buster - # compiler: g++ - # cpp: 17 - # asan: off - # ubsan: off - # light_runtime: on - - os: focal - compiler: clang++ + - os: buster + compiler: g++ cpp: 17 asan: off - ubsan: on + ubsan: off light_runtime: off - - os: focal + - os: buster + compiler: g++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on + - os: buster compiler: clang++ cpp: 17 asan: off - ubsan: on + ubsan: off light_runtime: on - os: focal - compiler: g++-10 - cpp: 20 - asan: on - ubsan: off + compiler: clang++ + cpp: 17 + asan: off + ubsan: on light_runtime: off - os: focal compiler: g++-10 cpp: 20 asan: on ubsan: off - light_runtime: on - - os: jammy - compiler: g++ - cpp: 20 - asan: on - ubsan: off light_runtime: off - os: jammy compiler: g++ cpp: 20 asan: on ubsan: off - light_runtime: on + light_runtime: off name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" diff --git a/.github/workflows/Dockerfile.buster b/.github/workflows/Dockerfile.buster index 908dbaed10..432855b3c9 100644 --- a/.github/workflows/Dockerfile.buster +++ b/.github/workflows/Dockerfile.buster @@ -8,8 +8,8 @@ RUN apt-get update && \ echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \ - wget -qO - https://packages.sury.org/php/apt.gpg | apt-key add - && \ - echo "deb https://packages.sury.org/php/ buster main" >> /etc/apt/sources.list.d/php.list && \ + wget -qO - https://debian.octopuce.fr/snapshots/sury-php/buster-latest/apt.gpg | apt-key add - && \ + echo "deb https://debian.octopuce.fr/snapshots/sury-php/buster-latest/ buster main" >> /etc/apt/sources.list && \ TEMP_DEB="$(mktemp)" && \ wget -O "$TEMP_DEB" 'https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb' && \ dpkg -i "$TEMP_DEB" && \ @@ -19,8 +19,8 @@ RUN apt-get update && \ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 && \ apt-get update && \ apt-get install -y --no-install-recommends \ - git cmake-data=3.18* cmake=3.18* make g++ gperf netcat \ - python3.7 python3-dev libpython3-dev python3-pip python3-setuptools mysql-server libmysqlclient-dev && \ + git cmake-data=3.18* cmake=3.18* make g++-10 clang++ gperf netcat \ + python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \ pip3 install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 1d365bd178..d4c9130881 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -28,6 +28,12 @@ jobs: asan: off ubsan: off light_runtime: on + - os: buster + compiler: clang++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 0280b01a48..49af16bd9f 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -21,39 +21,18 @@ jobs: cpp: 17 asan: off ubsan: off - light_runtime: off - - os: jammy - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: on - os: focal compiler: clang++ cpp: 17 asan: off ubsan: on - light_runtime: off - - os: focal - compiler: clang++ - cpp: 17 - asan: off - ubsan: on - light_runtime: on - - os: focal - compiler: g++-10 - cpp: 20 - asan: on - ubsan: off - light_runtime: off - os: focal compiler: g++-10 cpp: 20 asan: on ubsan: off - light_runtime: on - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" + name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" steps: - uses: actions/checkout@v3 From 08c5a047c4377aa07914d6dec521c8965aafa5d0 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Wed, 10 Jul 2024 11:34:33 +0300 Subject: [PATCH 52/89] Fix tags, change flock, add macros - add aliases with macros: getxattr(), setxattr() -> get_tag(), set_tag(), - change flock() -> fcntl() with F_SETLKW, - add an additional check for the validity of file descriptor in session_close() --- runtime/sessions.cpp | 87 ++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 31 deletions(-) diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index ffa31dcc4c..f587c1192f 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include "runtime/sessions.h" #include "runtime/files.h" @@ -41,7 +42,6 @@ constexpr static auto S_READ_CLOSE = "read_and_close"; constexpr static auto S_ID = "session_id"; constexpr static auto S_FD = "handler"; constexpr static auto S_STATUS = "session_status"; -constexpr static auto S_OPENED = "is_opened"; constexpr static auto S_DIR = "save_path"; constexpr static auto S_PATH = "session_path"; constexpr static auto S_NAME = "name"; @@ -71,6 +71,22 @@ const auto skeys = vk::to_array>({ {C_HTTPONLY, false} }); +static int set_tag(const char *path, const char *name, void *value, const size_t size) { +#if defined(__APPLE__) + return setxattr(path, name, value, size, 0, 0); +#else + return setxattr(path, name, value, size, 0); +#endif +} + +static int get_tag(const char *path, const char *name, void *value, const size_t size) { +#if defined(__APPLE__) + return getxattr(path, name, value, size, 0, 0); +#else + return getxattr(path, name, value, size); +#endif +} + static void initialize_sparams(const array &options) noexcept { for (const auto& it : skeys) { if (options.isset(string(it.first))) { @@ -174,11 +190,7 @@ static bool session_decode(const string &data) { } static bool session_open() { - if (get_sparam(S_FD).to_bool()) { - /* - if we close the file for reopening it, - the other worker may open it faster and the current process will stop - */ + if (get_sparam(S_FD).to_bool() && (fcntl(get_sparam(S_FD).to_int(), F_GETFD) != -1 || errno != EBADF)) { return true; } @@ -188,6 +200,8 @@ static bool session_open() { set_sparam(S_PATH, string(get_sparam(S_DIR).to_string()).append(get_sparam(S_ID).to_string())); bool is_new = (!f$file_exists(get_sparam(S_PATH).to_string())) ? 1 : 0; + + // the interprocessor lock does not work set_sparam(S_FD, open_safe(get_sparam(S_PATH).to_string().c_str(), O_RDWR | O_CREAT, 0777)); if (get_sparam(S_FD).to_int() < 0) { @@ -195,36 +209,40 @@ static bool session_open() { return false; } - if (flock(get_sparam(S_FD).to_int(), LOCK_EX) < 0) { - php_warning("Failed to lock the file %s", get_sparam(S_PATH).to_string().c_str()); - return false; - } + struct flock lock; + lock.l_type = F_WRLCK; // Exclusive write lock + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + lock.l_pid = getpid(); + + int ret = fcntl(get_sparam(S_FD).to_int(), F_SETLKW, &lock); + + if (ret < 0 && errno == EDEADLK) { + php_warning("Attempt to lock alredy locked session file, path: %s", get_sparam(S_PATH).to_string().c_str()); + return false; + } // set new metadata to the file - int ret_ctime = getxattr(get_sparam(S_PATH).to_string().c_str(), S_CTIME, NULL, 0, 0, 0); - int ret_gc_lifetime = getxattr(get_sparam(S_PATH).to_string().c_str(), S_LIFETIME, NULL, 0, 0, 0); + int ret_ctime = get_tag(get_sparam(S_PATH).to_string().c_str(), S_CTIME, NULL, 0); + int ret_gc_lifetime = get_tag(get_sparam(S_PATH).to_string().c_str(), S_LIFETIME, NULL, 0); if (is_new or ret_ctime < 0) { // add the creation data to metadata of file int ctime = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - setxattr(get_sparam(S_PATH).to_string().c_str(), S_CTIME, &ctime, sizeof(int), 0, 0); + set_tag(get_sparam(S_PATH).to_string().c_str(), S_CTIME, &ctime, sizeof(int)); } if (ret_gc_lifetime < 0) { int gc_maxlifetime = get_sparam(S_LIFETIME).to_int(); - setxattr(get_sparam(S_PATH).to_string().c_str(), S_LIFETIME, &gc_maxlifetime, sizeof(int), 0, 0); + set_tag(get_sparam(S_PATH).to_string().c_str(), S_LIFETIME, &gc_maxlifetime, sizeof(int)); } - int is_opened = 1; - setxattr(get_sparam(S_PATH).to_string().c_str(), S_OPENED, &is_opened, sizeof(int), 0, 0); - return true; } static void session_close() { - if (get_sparam(S_FD).to_bool()) { + if (get_sparam(S_FD).to_bool() && (fcntl(get_sparam(S_FD).to_int(), F_GETFD) != -1 || errno != EBADF)) { close_safe(get_sparam(S_FD).to_int()); } - int is_opened = 0; - setxattr(get_sparam(S_PATH).to_string().c_str(), S_OPENED, &is_opened, sizeof(int), 0, 0); reset_sparams(); } @@ -262,7 +280,7 @@ static bool session_read() { static bool session_write() { session_open(); string data = f$serialize(v$_SESSION.as_array()); - ssize_t n = write_safe(get_sparam("handler").to_int(), data.c_str(), data.size(), get_sparam("file_path").to_string()); + ssize_t n = write_safe(get_sparam(S_FD).to_int(), data.c_str(), data.size(), get_sparam(S_PATH).to_string()); if (n < data.size()) { if (n == -1) { php_warning("Write failed"); @@ -310,8 +328,8 @@ static bool session_reset_id() { static bool session_expired(const string &path) { int ctime, lifetime; - int ret_ctime = getxattr(path.c_str(), S_CTIME, &ctime, sizeof(int), 0, 0); - int ret_lifetime = getxattr(path.c_str(), S_LIFETIME, &lifetime, sizeof(int), 0, 0); + int ret_ctime = get_tag(path.c_str(), S_CTIME, &ctime, sizeof(int)); + int ret_lifetime = get_tag(path.c_str(), S_LIFETIME, &lifetime, sizeof(int)); if (ret_ctime < 0 or ret_lifetime < 0) { php_warning("Failed to get metadata of the file on path: %s", path.c_str()); return false; @@ -337,6 +355,13 @@ static int session_gc(const bool &immediate = false) { return -1; } + struct flock lock; + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + lock.l_pid = getpid(); + int result = 0; for (auto s = s_list.as_array().begin(); s != s_list.as_array().end(); ++s) { string path = s.get_value().to_string(); @@ -347,14 +372,15 @@ static int session_gc(const bool &immediate = false) { if (path == get_sparam(S_PATH).to_string()) { continue; } - int is_opened; - int ret_code = getxattr(path.c_str(), S_OPENED, &is_opened, sizeof(int), 0, 0); - if (ret_code < 0) { - php_warning("Failed to get metadata of the file on path: %s", path.c_str()); + + int fd; + if ((fd = open(path.c_str(), O_RDWR, 0777)) < 0) { + php_warning("Failed to open file on path: %s", path.c_str()); + continue; + } + + if (fcntl(fd, F_SETLK, &lock) < 0) { continue; - } else if (is_opened) { - // TO-DO: fix the bug with always opened tags in the session files - continue; } if (session_expired(path)) { @@ -507,7 +533,6 @@ Optional f$session_id(const Optional &id) { php_warning("Session ID cannot be changed when a session is active"); return Optional{false}; } - mixed prev_id = sessions::get_sparam(sessions::S_ID); if (id.has_value()) { sessions::set_sparam(sessions::S_ID, id.val()); From b2ecb61bcbf97a3c27c81d6d0436e4e626c32839 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Wed, 10 Jul 2024 23:24:06 +0300 Subject: [PATCH 53/89] Add cpp-tests (within single process (worker)) - session_id(), - session_start(), - session_status() --- tests/cpp/runtime/runtime-tests.cmake | 3 +- tests/cpp/runtime/sessions-test.cpp | 54 +++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tests/cpp/runtime/sessions-test.cpp diff --git a/tests/cpp/runtime/runtime-tests.cmake b/tests/cpp/runtime/runtime-tests.cmake index 731da98fe6..1466b1a33d 100644 --- a/tests/cpp/runtime/runtime-tests.cmake +++ b/tests/cpp/runtime/runtime-tests.cmake @@ -21,7 +21,8 @@ prepend(RUNTIME_TESTS_SOURCES ${BASE_DIR}/tests/cpp/runtime/ memory_resource/unsynchronized_pool_resource-test.cpp string-list-test.cpp string-test.cpp - zstd-test.cpp) + zstd-test.cpp + sessions-test.cpp) allow_deprecated_declarations_for_apple(${BASE_DIR}/tests/cpp/runtime/inter-process-mutex-test.cpp) vk_add_unittest(runtime "${RUNTIME_LIBS};${RUNTIME_LINK_TEST_LIBS}" ${RUNTIME_TESTS_SOURCES}) diff --git a/tests/cpp/runtime/sessions-test.cpp b/tests/cpp/runtime/sessions-test.cpp new file mode 100644 index 0000000000..2354389912 --- /dev/null +++ b/tests/cpp/runtime/sessions-test.cpp @@ -0,0 +1,54 @@ +#include + +#include "runtime/files.h" +#include "runtime/sessions.h" + +TEST(sessions_test, test_session_id_with_invalid_id) { + const string dir_path = string(getenv("TMPDIR")).append("sessions/"); + const string id = string("\t12345678\\\r"); + ASSERT_FALSE(f$session_id(id).has_value()); + ASSERT_TRUE(f$session_start()); + ASSERT_NE(f$session_id().val(), id); + + const string full_path = string(dir_path).append(f$session_id().val()); + ASSERT_TRUE(f$file_exists(full_path)); + ASSERT_TRUE(f$session_abort()); +} + +TEST(sessions_test, test_session_id_with_valid_id) { + const string dir_path = string(getenv("TMPDIR")).append("sessions/"); + const string id = string("sess_668d4f818ca3b"); + ASSERT_FALSE(f$session_id(id).has_value()); + ASSERT_TRUE(f$session_start()); + ASSERT_EQ(f$session_id().val(), id); + + const string full_path = string(dir_path).append(id); + ASSERT_TRUE(f$file_exists(full_path)); + ASSERT_TRUE(f$session_abort()); +} + +TEST(sessions_test, test_session_start) { + ASSERT_TRUE(f$session_start()); + ASSERT_FALSE(f$session_start()); + ASSERT_TRUE(f$session_abort()); +} + +TEST(sessions_test, test_session_start_with_params) { + array predefined_consts = array(); + const string dir_path = string(getenv("TMPDIR")).append("example/"); + predefined_consts.emplace_value(string("save_path"), dir_path); + ASSERT_TRUE(f$session_start(predefined_consts)); + ASSERT_TRUE(f$file_exists(string(dir_path).append(f$session_id().val()))); + ASSERT_TRUE(f$session_abort()); +} + +TEST(sessions_test, test_session_status) { + const int SESSION_NONE = 1; + const int SESSION_ACTIVE = 2; + + ASSERT_EQ(f$session_status(), SESSION_NONE); + ASSERT_TRUE(f$session_start()); + ASSERT_EQ(f$session_status(), SESSION_ACTIVE); + ASSERT_TRUE(f$session_abort()); + ASSERT_EQ(f$session_status(), SESSION_NONE); +} \ No newline at end of file From aeb540ef73645b2d200e9c478378c667bfba46ae Mon Sep 17 00:00:00 2001 From: Andrey Arutiunian <110744283+andreylzmw@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:38:23 +0300 Subject: [PATCH 54/89] sury issue (#1036) --- .github/workflows/Dockerfile.buster | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Dockerfile.buster b/.github/workflows/Dockerfile.buster index 908dbaed10..c19dd5ef5a 100644 --- a/.github/workflows/Dockerfile.buster +++ b/.github/workflows/Dockerfile.buster @@ -8,8 +8,8 @@ RUN apt-get update && \ echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \ - wget -qO - https://packages.sury.org/php/apt.gpg | apt-key add - && \ - echo "deb https://packages.sury.org/php/ buster main" >> /etc/apt/sources.list.d/php.list && \ + wget -qO - https://debian.octopuce.fr/snapshots/sury-php/buster-latest/apt.gpg | apt-key add - && \ + echo "deb https://debian.octopuce.fr/snapshots/sury-php/buster-latest/ buster main" >> /etc/apt/sources.list && \ TEMP_DEB="$(mktemp)" && \ wget -O "$TEMP_DEB" 'https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb' && \ dpkg -i "$TEMP_DEB" && \ @@ -20,7 +20,7 @@ RUN apt-get update && \ apt-get update && \ apt-get install -y --no-install-recommends \ git cmake-data=3.18* cmake=3.18* make g++ gperf netcat \ - python3.7 python3-dev libpython3-dev python3-pip python3-setuptools mysql-server libmysqlclient-dev && \ + python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \ pip3 install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ From 6826f8b5105a21274afcc41eb31de5ce52feaea7 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Fri, 12 Jul 2024 13:31:21 +0300 Subject: [PATCH 55/89] Add help messages for debagging, fix the session_write() with rewind fd logic --- runtime/sessions.cpp | 71 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index f587c1192f..89fa3a51cf 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -169,6 +169,7 @@ static bool session_generate_id() { } static bool session_abort() { + fprintf(stderr, "[%d]->sessions::session_abort()\n", getpid()); if (get_sparam(S_STATUS).to_bool()) { session_close(); return true; @@ -177,12 +178,16 @@ static bool session_abort() { } static string session_encode() { + fprintf(stderr, "[%d]->sessions::session_encode()\n", getpid()); return f$serialize(v$_SESSION.as_array()); } static bool session_decode(const string &data) { + fprintf(stderr, "[%d]->sessions::session_decode()\n", getpid()); + fprintf(stderr, "[%d]\tsession_decode(): Call unserialize() for session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); mixed buf = f$unserialize(data); if (buf.is_bool()) { + fprintf(stderr, "[%d]\tsession_decode(): unserialize() returned false\n", getpid()); return false; } v$_SESSION = buf.as_array(); @@ -190,7 +195,9 @@ static bool session_decode(const string &data) { } static bool session_open() { + fprintf(stderr, "[%d]->sessions::session_open()\n", getpid()); if (get_sparam(S_FD).to_bool() && (fcntl(get_sparam(S_FD).to_int(), F_GETFD) != -1 || errno != EBADF)) { + fprintf(stderr, "[%d]\tSession id already opened, skip: %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); return true; } @@ -201,13 +208,15 @@ static bool session_open() { set_sparam(S_PATH, string(get_sparam(S_DIR).to_string()).append(get_sparam(S_ID).to_string())); bool is_new = (!f$file_exists(get_sparam(S_PATH).to_string())) ? 1 : 0; - // the interprocessor lock does not work + fprintf(stderr, "[%d]\tOpening the session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); set_sparam(S_FD, open_safe(get_sparam(S_PATH).to_string().c_str(), O_RDWR | O_CREAT, 0777)); if (get_sparam(S_FD).to_int() < 0) { php_warning("Failed to open the file %s", get_sparam(S_PATH).to_string().c_str()); + fprintf(stderr, "[%d]\tFailed to open the session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); return false; } + fprintf(stderr, "[%d]\tSuccessfully opened the session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); struct flock lock; lock.l_type = F_WRLCK; // Exclusive write lock @@ -216,12 +225,15 @@ static bool session_open() { lock.l_len = 0; lock.l_pid = getpid(); + fprintf(stderr, "[%d]\tLocking the session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); int ret = fcntl(get_sparam(S_FD).to_int(), F_SETLKW, &lock); if (ret < 0 && errno == EDEADLK) { php_warning("Attempt to lock alredy locked session file, path: %s", get_sparam(S_PATH).to_string().c_str()); + fprintf(stderr, "[%d]\tFailed to lock the session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); return false; } + fprintf(stderr, "[%d]\tSuccessfully lock the session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); // set new metadata to the file int ret_ctime = get_tag(get_sparam(S_PATH).to_string().c_str(), S_CTIME, NULL, 0); @@ -240,13 +252,27 @@ static bool session_open() { } static void session_close() { + fprintf(stderr, "[%d]->sessions::session_close()\n", getpid()); if (get_sparam(S_FD).to_bool() && (fcntl(get_sparam(S_FD).to_int(), F_GETFD) != -1 || errno != EBADF)) { + struct flock lock; + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + lock.l_pid = getpid(); + + fprintf(stderr, "[%d]\tUnlocking the session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); + fcntl(get_sparam(S_FD).to_int(), F_SETLKW, &lock); + fprintf(stderr, "[%d]\tUnlocked the session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); + fprintf(stderr, "[%d]\tNow closing the session file %s\n\n", getpid(), get_sparam(S_PATH).to_string().c_str()); close_safe(get_sparam(S_FD).to_int()); } + set_sparam(S_STATUS, false); reset_sparams(); } static bool session_read() { + fprintf(stderr, "[%d]->sessions::session_read()\n", getpid()); session_open(); struct stat buf; if (fstat(get_sparam(S_FD).to_int(), &buf) < 0) { @@ -269,6 +295,8 @@ static bool session_read() { } return false; } + + fprintf(stderr, "[%d]\tsession_read(): Successfully read the session file, result:\n\t%s\n", getpid(), string(result, n).c_str()); if (!session_decode(string(result, n))) { php_warning("Failed to unzerialize the data"); @@ -278,8 +306,33 @@ static bool session_read() { } static bool session_write() { - session_open(); + fprintf(stderr, "[%d]->sessions::session_write()\n", getpid()); + + // rewind the fd + fprintf(stderr, "[%d]\trewinding the S_FD\n", getpid()); + struct flock lock; + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + lock.l_pid = getpid(); + fprintf(stderr, "[%d]\t(rewinding) Unlocking the file before closing\n", getpid()); + fcntl(get_sparam(S_FD).to_int(), F_SETLKW, &lock); + fprintf(stderr, "[%d]\t(rewinding) Unlocked the file before closing\n", getpid()); + close_safe(get_sparam(S_FD).to_int()); + fprintf(stderr, "[%d]\t(rewinding) Closed the file\n", getpid()); + + fprintf(stderr, "[%d]\t(rewinding) Opening the file\n", getpid()); + set_sparam(S_FD, open_safe(get_sparam(S_PATH).to_string().c_str(), O_RDWR, 0777)); + fprintf(stderr, "[%d]\t(rewinding) Opened the file\n", getpid()); + lock.l_type = F_WRLCK; + fprintf(stderr, "[%d]\t(rewinding) Locking the file\n", getpid()); + fcntl(get_sparam(S_FD).to_int(), F_SETLKW, &lock); + fprintf(stderr, "[%d]\t(rewinding) Locked the file\n", getpid()); + fprintf(stderr, "[%d]\tSuccessfully rewind the S_FD: %s\n", getpid(), get_sparam(S_ID).to_string().c_str()); + string data = f$serialize(v$_SESSION.as_array()); + ssize_t n = write_safe(get_sparam(S_FD).to_int(), data.c_str(), data.size(), get_sparam(S_PATH).to_string()); if (n < data.size()) { if (n == -1) { @@ -289,6 +342,7 @@ static bool session_write() { } return false; } + fprintf(stderr, "[%d]\tsession_write() completed\n", getpid()); return true; } @@ -392,6 +446,7 @@ static int session_gc(const bool &immediate = false) { } static bool session_initialize() { + fprintf(stderr, "[%d]->sessions::session_initialize()\n", getpid()); set_sparam(S_STATUS, true); if (!get_sparam(S_ID).to_bool()) { @@ -418,6 +473,7 @@ static bool session_initialize() { } static bool session_start() { + fprintf(stderr, "[%d]->sessions::session_start()\n", getpid()); if (get_sparam(S_STATUS).to_bool()) { php_warning("Ignoring session_start() because a session is already active"); return false; @@ -453,6 +509,7 @@ static bool session_start() { } static bool session_flush() { + fprintf(stderr, "[%d]->sessions::session_flush()\n", getpid()); if (!get_sparam(S_STATUS).to_bool()) { return false; } @@ -465,6 +522,7 @@ static bool session_flush() { } // namespace sessions bool f$session_start(const array &options) { + fprintf(stderr, "[%d]->f$session_start()\n", getpid()); if (sessions::get_sparam(sessions::S_STATUS).to_bool()) { php_warning("Ignoring session_start() because a session is already active"); return false; @@ -480,6 +538,7 @@ bool f$session_start(const array &options) { } bool f$session_abort() { + fprintf(stderr, "[%d]->f$session_abort()\n", getpid()); if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { return false; } @@ -488,6 +547,7 @@ bool f$session_abort() { } Optional f$session_gc() { + fprintf(stderr, "[%d]->f$session_gc()\n", getpid()); if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { php_warning("Session cannot be garbage collected when there is no active session"); return false; @@ -505,6 +565,7 @@ bool f$session_write_close() { } bool f$session_commit() { + fprintf(stderr, "[%d]->f$session_commit()\n", getpid()); return f$session_write_close(); } @@ -517,6 +578,7 @@ Optional f$session_encode() { } bool f$session_decode(const string &data) { + fprintf(stderr, "[%d]->f$session_decode()\n", getpid()); if (!sessions::get_sparam(sessions::S_STATUS).to_bool()) { php_warning("Session data cannot be decoded when there is no active session"); return false; @@ -529,15 +591,18 @@ array f$session_get_cookie_params() { } Optional f$session_id(const Optional &id) { + fprintf(stderr, "[%d]->f$session_id()\n", getpid()); if (id.has_value() && sessions::get_sparam(sessions::S_STATUS).to_bool()) { php_warning("Session ID cannot be changed when a session is active"); return Optional{false}; } mixed prev_id = sessions::get_sparam(sessions::S_ID); if (id.has_value()) { + fprintf(stderr, "[%d]\tGot an id: %s\n", getpid(), id.val().c_str()); sessions::set_sparam(sessions::S_ID, id.val()); + fprintf(stderr, "[%d]\tSet the id: %s\n", getpid(), sessions::get_sparam(sessions::S_ID).to_string().c_str()); } - return (prev_id.is_bool()) ? Optional{false} : Optional(prev_id.as_string()); + return (prev_id.is_bool()) ? Optional{false} : Optional(prev_id.to_string()); } // TO-DO From 755e510b3953322c9b547348f143dce802148ccc Mon Sep 17 00:00:00 2001 From: Alexander Polyakov Date: Fri, 12 Jul 2024 17:35:40 +0300 Subject: [PATCH 56/89] Disable large strings in RPC fetch and store (#1040) --- runtime-light/stdlib/rpc/rpc-api.cpp | 69 +++++++++++++++------------- runtime-light/streams/interface.cpp | 4 +- runtime-light/streams/interface.h | 2 + 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/runtime-light/stdlib/rpc/rpc-api.cpp b/runtime-light/stdlib/rpc/rpc-api.cpp index 48e3bf9ed4..b7ca4f4d0a 100644 --- a/runtime-light/stdlib/rpc/rpc-api.cpp +++ b/runtime-light/stdlib/rpc/rpc-api.cpp @@ -24,7 +24,10 @@ namespace rpc_impl_ { constexpr int32_t MAX_TIMEOUT_S = 86400; -// TODO: change uint64_t to string::size_type after moving it from uint32_t to uint64_t +constexpr auto SMALL_STRING_SIZE_LEN = 1; +constexpr auto MEIDUM_STRING_SIZE_LEN = 3; +constexpr auto LARGE_STRING_SIZE_LEN = 7; + constexpr uint64_t SMALL_STRING_MAX_LEN = 253; constexpr uint64_t MEDIUM_STRING_MAX_LEN = (static_cast(1) << 24) - 1; [[maybe_unused]] constexpr uint64_t LARGE_STRING_MAX_LEN = (static_cast(1) << 56) - 1; @@ -324,30 +327,25 @@ bool f$store_double(double v) noexcept { return true; } -bool f$store_string(const string &v) noexcept { // TODO: support large strings +bool f$store_string(const string &v) noexcept { auto &rpc_buf{RpcComponentContext::get().rpc_buffer}; - string::size_type string_len{v.size()}; - string::size_type size_len{}; + uint8_t size_len{}; + uint64_t string_len{v.size()}; if (string_len <= rpc_impl_::SMALL_STRING_MAX_LEN) { - size_len = 1; + size_len = rpc_impl_::SMALL_STRING_SIZE_LEN; rpc_buf.store_trivial(static_cast(string_len)); } else if (string_len <= rpc_impl_::MEDIUM_STRING_MAX_LEN) { - size_len = 4; + size_len = rpc_impl_::MEIDUM_STRING_SIZE_LEN + 1; rpc_buf.store_trivial(static_cast(rpc_impl_::MEDIUM_STRING_MAGIC)); rpc_buf.store_trivial(static_cast(string_len & 0xff)); rpc_buf.store_trivial(static_cast((string_len >> 8) & 0xff)); rpc_buf.store_trivial(static_cast((string_len >> 16) & 0xff)); } else { - size_len = 8; - rpc_buf.store_trivial(static_cast(rpc_impl_::LARGE_STRING_MAGIC)); - rpc_buf.store_trivial(static_cast(string_len & 0xff)); - rpc_buf.store_trivial(static_cast((string_len >> 8) & 0xff)); - rpc_buf.store_trivial(static_cast((string_len >> 16) & 0xff)); - rpc_buf.store_trivial(static_cast((string_len >> 24) & 0xff)); - // rpc_buf.store_trivial(static_cast((string_len >> 32) & 0xff)); - // rpc_buf.store_trivial(static_cast((string_len >> 40) & 0xff)); - // rpc_buf.store_trivial(static_cast((string_len >> 48) & 0xff)); + php_warning("large strings aren't supported"); + size_len = rpc_impl_::SMALL_STRING_SIZE_LEN; + string_len = 0; + rpc_buf.store_trivial(static_cast(string_len)); } rpc_buf.store(v.c_str(), string_len); @@ -388,14 +386,14 @@ string f$fetch_string() noexcept { return {}; // TODO: error handling } - string::size_type string_len{}; - string::size_type size_len{}; + uint8_t size_len{}; + uint64_t string_len{}; switch (first_byte) { - case rpc_impl_::LARGE_STRING_MAGIC: { // next 7 bytes are string's length // TODO: support large strings - // static_assert(sizeof(string::size_type) >= 8, "string's length doesn't fit platform size"); - if (rpc_buf.remaining() < 7) { + case rpc_impl_::LARGE_STRING_MAGIC: { + if (rpc_buf.remaining() < rpc_impl_::LARGE_STRING_SIZE_LEN) { return {}; // TODO: error handling } + size_len = rpc_impl_::LARGE_STRING_SIZE_LEN + 1; const auto first{static_cast(rpc_buf.fetch_trivial().value())}; const auto second{static_cast(rpc_buf.fetch_trivial().value()) << 8}; const auto third{static_cast(rpc_buf.fetch_trivial().value()) << 16}; @@ -404,35 +402,40 @@ string f$fetch_string() noexcept { const auto sixth{static_cast(rpc_buf.fetch_trivial().value()) << 40}; const auto seventh{static_cast(rpc_buf.fetch_trivial().value()) << 48}; string_len = first | second | third | fourth | fifth | sixth | seventh; - if (string_len < (1 << 24)) { - php_warning("long string's length is less than 1 << 24"); - } - size_len = 8; + + const auto total_len_with_padding{(size_len + string_len + 3) & ~static_cast(3)}; + rpc_buf.adjust(total_len_with_padding - size_len); + php_warning("large strings aren't supported"); + return {}; } - case rpc_impl_::MEDIUM_STRING_MAGIC: { // next 3 bytes are string's length - if (rpc_buf.remaining() < 3) { + case rpc_impl_::MEDIUM_STRING_MAGIC: { + if (rpc_buf.remaining() < rpc_impl_::MEIDUM_STRING_SIZE_LEN) { return {}; // TODO: error handling } + size_len = rpc_impl_::MEIDUM_STRING_SIZE_LEN + 1; const auto first{static_cast(rpc_buf.fetch_trivial().value())}; const auto second{static_cast(rpc_buf.fetch_trivial().value()) << 8}; const auto third{static_cast(rpc_buf.fetch_trivial().value()) << 16}; string_len = first | second | third; - if (string_len <= 253) { + + if (string_len <= rpc_impl_::SMALL_STRING_MAX_LEN) { php_warning("long string's length is less than 254"); } - size_len = 4; + break; + } + default: { + size_len = rpc_impl_::SMALL_STRING_SIZE_LEN; + string_len = static_cast(first_byte); + break; } - default: - string_len = static_cast(first_byte); - size_len = 1; } - const auto total_len_with_padding{(size_len + string_len + 3) & ~static_cast(3)}; + const auto total_len_with_padding{(size_len + string_len + 3) & ~static_cast(3)}; if (rpc_buf.remaining() < total_len_with_padding - size_len) { return {}; // TODO: error handling } - string res{rpc_buf.data() + rpc_buf.pos(), string_len}; + string res{rpc_buf.data() + rpc_buf.pos(), static_cast(string_len)}; rpc_buf.adjust(total_len_with_padding - size_len); return res; } diff --git a/runtime-light/streams/interface.cpp b/runtime-light/streams/interface.cpp index 66ecc70ae0..ad9fddf1cc 100644 --- a/runtime-light/streams/interface.cpp +++ b/runtime-light/streams/interface.cpp @@ -21,7 +21,7 @@ task_t f$component_get_http_query() { task_t> f$component_client_send_query(const string &name, const string &message) { class_instance query; const PlatformCtx &ptx = *get_platform_context(); - uint64_t stream_d; + uint64_t stream_d{}; OpenStreamResult res = ptx.open(name.size(), name.c_str(), &stream_d); if (res != OpenStreamOk) { php_warning("cannot open stream"); @@ -94,7 +94,7 @@ class_instance f$component_open_stream(const string &name) { class_instance query; const PlatformCtx &ptx = *get_platform_context(); ComponentState &ctx = *get_component_context(); - uint64_t stream_d; + uint64_t stream_d{}; OpenStreamResult res = ptx.open(name.size(), name.c_str(), &stream_d); if (res != OpenStreamOk) { php_warning("cannot open stream"); diff --git a/runtime-light/streams/interface.h b/runtime-light/streams/interface.h index 400b0aecda..8e48b67539 100644 --- a/runtime-light/streams/interface.h +++ b/runtime-light/streams/interface.h @@ -4,6 +4,8 @@ #pragma once +#include + #include "runtime-light/coroutine/task.h" #include "runtime-light/streams/component-stream.h" From 7da9cce945f427dbb0dd9f8b5d331bab30bf4a1b Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Fri, 12 Jul 2024 18:57:05 +0300 Subject: [PATCH 57/89] Add github action test --- .github/workflows/Build.yml | 166 ++++++++++++++----------- .github/workflows/Dockerfile.buster | 4 +- docs/session_test.php | 185 ++++++++++++++++++++++++++++ 3 files changed, 283 insertions(+), 72 deletions(-) create mode 100644 docs/session_test.php diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index d3c8d31e5f..723cad1972 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -11,6 +11,9 @@ env: kphp_root_dir: /home/kitten/kphp kphp_polyfills_dir: /home/kitten/kphp/kphp-polyfills kphp_build_dir: /home/kitten/kphp/build + session_php_script: /home/kitten/kphp/docs/session_test.php + kphp_snippets_dir: /home/kitten/kphp/kphp-snippets + kphp_snippets_jobs_dir: /home/kitten/kphp/kphp-snippets/JobWorkers jobs: build-linux: @@ -18,11 +21,11 @@ jobs: strategy: matrix: include: - - os: buster - compiler: g++ - cpp: 17 - asan: off - ubsan: off + # - os: buster + # compiler: g++ + # cpp: 17 + # asan: off + # ubsan: off - os: focal compiler: clang++ cpp: 17 @@ -50,6 +53,12 @@ jobs: repository: 'VKCOM/kphp-polyfills' path: 'kphp-polyfills' + - name: Get snippets repo + uses: actions/checkout@v3 + with: + repository: 'VKCOM/kphp-snippets' + path: 'kphp-snippets' + - name: Cache docker image uses: actions/cache@v3 id: docker-image-cache @@ -80,82 +89,99 @@ jobs: run: docker exec kphp-build-container-${{matrix.os}} bash -c "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" - - name: Run unit tests - run: docker exec kphp-build-container-${{matrix.os}} bash -c - "make -C ${{env.kphp_build_dir}} -j$(nproc) test" + # - name: Run unit tests + # run: docker exec kphp-build-container-${{matrix.os}} bash -c + # "make -C ${{env.kphp_build_dir}} -j$(nproc) test" - - name: Compile dummy PHP script - run: docker exec kphp-build-container-${{matrix.os}} bash -c - "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php && kphp_out/server -o --user kitten" + # - name: Compile dummy PHP script + # run: docker exec kphp-build-container-${{matrix.os}} bash -c + # "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php && kphp_out/server -o --user kitten" - - name: Polyfills composer install - run: docker exec kphp-build-container-${{matrix.os}} bash -c - "composer install -d ${{env.kphp_polyfills_dir}}" + # - name: Polyfills composer install + # run: docker exec kphp-build-container-${{matrix.os}} bash -c + # "composer install -d ${{env.kphp_polyfills_dir}}" - - name: Run python tests - id: python_tests - continue-on-error: true - run: docker exec kphp-build-container-${{matrix.os}} bash -c - "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'" + # - name: Run python tests + # id: python_tests + # continue-on-error: true + # run: docker exec kphp-build-container-${{matrix.os}} bash -c + # "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'" - - name: Prepare python tests artifacts - if: steps.python_tests.outcome == 'failure' - run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} && - rm -rf ${{runner.temp}}/_tmp/*/working_dir + # - name: Prepare python tests artifacts + # if: steps.python_tests.outcome == 'failure' + # run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} && + # rm -rf ${{runner.temp}}/_tmp/*/working_dir - - name: Upload python tests artifacts - uses: actions/upload-artifact@v3 - if: steps.python_tests.outcome == 'failure' - with: - path: ${{runner.temp}}/_tmp/ + # - name: Upload python tests artifacts + # uses: actions/upload-artifact@v3 + # if: steps.python_tests.outcome == 'failure' + # with: + # path: ${{runner.temp}}/_tmp/ - - name: Fail pipeline if python tests failed - if: steps.python_tests.outcome == 'failure' - run: exit 1 + # - name: Fail pipeline if python tests failed + # if: steps.python_tests.outcome == 'failure' + # run: exit 1 - - name: Remove docker container - run: docker rm -f kphp-build-container-${{matrix.os}} + - name: Copy the session_test file + run: docker exec kphp-build-container-${{matrix.os}} bash -c + "su kitten -c 'cp ${{env.session_php_script}} ${{env.kphp_snippets_dir}}/JobWorkers/'" - build-macos: - runs-on: ${{matrix.os}}-12 - strategy: - matrix: - include: - - os: macos - compiler: clang++ - cpp: 17 + - name: Compile the session_test project + run: docker exec kphp-build-container-${{matrix.os}} bash -c + "su kitten -c 'cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} session_test.php -I .. -M server'" - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}" - - steps: - - uses: actions/checkout@v3 + - name: Start the server + run: docker exec kphp-build-container-${{matrix.os}} bash -c + "su kitten -c 'cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ./kphp_out/server -f 2 --http-port 8080 --job-workers-ratio 0.5 --sql-port 5555 &> log.txt &'" - # because of https://github.com/orgs/Homebrew/discussions/4612 - - name: Check Environment - run: | - export HOMEBREW_NO_INSTALL_FROM_API=0 - brew untap --force homebrew/cask + - name: Send a request to the server + run: docker exec kphp-build-container-${{matrix.os}} bash -c + "curl 'http://localhost:8080'" - - name: Setup Environment - run: | - brew tap shivammathur/php - brew update - brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4 - brew link --overwrite --force shivammathur/php/php@7.4 - /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install jsonschema install --break-system-packages jsonschema + # - name: Remove docker container + # run: docker rm -f kphp-build-container-${{matrix.os}} - - name: Run cmake - run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build - - name: Build all - run: make -C ${{runner.workspace}}/build -j$(nproc) all - - - name: Run unit tests - run: make -C ${{runner.workspace}}/build -j$(nproc) test + # build-macos: + # runs-on: ${{matrix.os}}-12 + # strategy: + # matrix: + # include: + # - os: macos + # compiler: clang++ + # cpp: 17 - - name: Compile dummy PHP script - working-directory: ${{runner.workspace}}/build - run: | - echo 'hello world' > demo.php - $GITHUB_WORKSPACE/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php - kphp_out/server -o + # name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}" + + # steps: + # - uses: actions/checkout@v3 + + # # because of https://github.com/orgs/Homebrew/discussions/4612 + # - name: Check Environment + # run: | + # export HOMEBREW_NO_INSTALL_FROM_API=0 + # brew untap --force homebrew/cask + + # - name: Setup Environment + # run: | + # brew tap shivammathur/php + # brew update + # brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4 + # brew link --overwrite --force shivammathur/php/php@7.4 + # /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install jsonschema install --break-system-packages jsonschema + + # - name: Run cmake + # run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build + + # - name: Build all + # run: make -C ${{runner.workspace}}/build -j$(nproc) all + + # - name: Run unit tests + # run: make -C ${{runner.workspace}}/build -j$(nproc) test + + # - name: Compile dummy PHP script + # working-directory: ${{runner.workspace}}/build + # run: | + # echo 'hello world' > demo.php + # $GITHUB_WORKSPACE/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php + # kphp_out/server -o diff --git a/.github/workflows/Dockerfile.buster b/.github/workflows/Dockerfile.buster index 09f667fd8d..acbec55660 100644 --- a/.github/workflows/Dockerfile.buster +++ b/.github/workflows/Dockerfile.buster @@ -8,8 +8,8 @@ RUN apt-get update && \ echo "deb https://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \ - wget -qO - https://packages.sury.org/php/apt.gpg | apt-key add - && \ - echo "deb https://packages.sury.org/php/ buster main" >> /etc/apt/sources.list.d/php.list && \ + wget -qO - https://debian.octopuce.fr/snapshots/sury-php/buster-latest/apt.gpg | apt-key add - && \ + echo "deb https://debian.octopuce.fr/snapshots/sury-php/buster-latest/ buster main" >> /etc/apt/sources.list && \ TEMP_DEB="$(mktemp)" && \ wget -O "$TEMP_DEB" 'https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb' && \ dpkg -i "$TEMP_DEB" && \ diff --git a/docs/session_test.php b/docs/session_test.php new file mode 100644 index 0000000000..b82e8cab7e --- /dev/null +++ b/docs/session_test.php @@ -0,0 +1,185 @@ + */ + public $predefined_consts; + + /** @var mixed */ + public $session_array; + + /** @var bool */ + public $must_sleep; + + /** + * @param array $predefined_consts + * @param mixed $session_array + * @param false|string $id + * @param bool $must_sleep + */ + public function __construct($predefined_consts, $session_array, $id, $must_sleep) { + $this->session_id = $id; + $this->predefined_consts = $predefined_consts; + $this->session_array = $session_array; + $this->must_sleep = $must_sleep; + } + + function handleRequest(): ?\KphpJobWorkerResponse { + $response = new MyResponse(); + session_id($this->session_id); + + $response->start_time = microtime(true); + session_start($this->predefined_consts); + $response->session_status = (session_status() == 2); + $response->end_time = microtime(true); + + $response->session_id = session_id(); + if (!$response->session_status) { + return $response; + } + + $session_array_before = unserialize(session_encode()); + session_decode(serialize(array_merge($session_array_before, $this->session_array))); + // or: $_SESSION = array_merge($this->session_array, $_SESSION); + $response->session_array = unserialize(session_encode()); + // or: $response->session_array = $_SESSION; + + if ($this->must_sleep) { + usleep(2 * 100000); + } + session_commit(); + + return $response; + } +} + +class MyResponse implements \KphpJobWorkerResponse { + /** @var float */ + public $start_time; + + /** @var float */ + public $end_time; + + /** @var bool */ + public $session_status; + + /** @var string|false */ + public $session_id; + + /** @var mixed */ + public $session_array; +} + + +if (PHP_SAPI !== 'cli' && isset($_SERVER["JOB_ID"])) { + handleKphpJobWorkerRequest(); +} else { + handleHttpRequest(); +} + +function handleHttpRequest() { + if (!\JobWorkers\JobLauncher::isEnabled()) { + echo "JOB WORKERS DISABLED at server start, use -f 2 --job-workers-ratio 0.5", "\n"; + return; + } + + $timeout = 0.5; + $to_write = ["first_message" => "hello"]; + + // ["save_path" => "/Users/marat/Desktop/sessions/"] + $main_request = new MyRequest([], $to_write, false, false); + $main_job_id = \JobWorkers\JobLauncher::start($main_request, $timeout); + + $main_response = wait($main_job_id); + $session_id = false; + + if ($main_response instanceof MyResponse) { + echo "Created main session:
"; + var_dump($main_response->session_status); + var_dump($main_response->session_id); + var_dump($main_response->session_array); + $session_id = $main_response->session_id; + echo "

"; + } else { + return; + } + + $main_request = new MyRequest([], ["second_message" => "world"], $session_id, true); + $main_job_id = \JobWorkers\JobLauncher::start($main_request, $timeout); + + $new_request = new MyRequest([], ["third_message" => "buy"], $session_id, false); + $new_job_id = \JobWorkers\JobLauncher::start($new_request, $timeout); + + $new_response = wait($new_job_id); + $main_response = wait($main_job_id); + + if ($main_response instanceof MyResponse) { + echo "
Opened session:
"; + var_dump($main_response->session_status); + var_dump($main_response->session_id); + var_dump($main_response->session_array); + echo "

"; + } + + if ($new_response instanceof MyResponse) { + echo "
Opened session:
"; + var_dump($new_response->session_status); + var_dump($new_response->session_id); + var_dump($new_response->session_array); + echo "

"; + } +} + +function handleKphpJobWorkerRequest() { + $kphp_job_request = kphp_job_worker_fetch_request(); + if (!$kphp_job_request) { + warning("Couldn't fetch a job worker request"); + return; + } + + if ($kphp_job_request instanceof \JobWorkers\JobWorkerSimple) { + // simple jobs: they start, finish, and return the result + $kphp_job_request->beforeHandle(); + $response = $kphp_job_request->handleRequest(); + if ($response === null) { + warning("Job request handler returned null for " . get_class($kphp_job_request)); + return; + } + kphp_job_worker_store_response($response); + + } else if ($kphp_job_request instanceof \JobWorkers\JobWorkerManualRespond) { + // more complicated jobs: they start, send a result in the middle (here get get it) — and continue working + $kphp_job_request->beforeHandle(); + $kphp_job_request->handleRequest(); + if (!$kphp_job_request->wasResponded()) { + warning("Job request handler didn't call respondAndContinueExecution() manually " . get_class($kphp_job_request)); + } + + } else if ($kphp_job_request instanceof \JobWorkers\JobWorkerNoReply) { + // background jobs: they start and never send any result, just continue in the background and finish somewhen + $kphp_job_request->beforeHandle(); + $kphp_job_request->handleRequest(); + + } else { + warning("Got unexpected job request class: " . get_class($kphp_job_request)); + } +} From 99b437f9b176582000dead188de859e7f1b34b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 11 Jul 2024 06:14:58 +0300 Subject: [PATCH 58/89] focal+gcc-10 and buster+clang --- .github/workflows/Build.yml | 16 +++++++-------- .github/workflows/Dockerfile | 32 +++++++++++++++++++++++++++++ .github/workflows/Dockerfile.buster | 7 +++++-- .github/workflows/Dockerfile.focal | 2 +- 4 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/Dockerfile diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 14c8941479..1a8a23f353 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -25,14 +25,8 @@ jobs: ubsan: off light_runtime: off - os: buster - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: on - - os: buster - compiler: clang++ - cpp: 17 + compiler: clang++-17 + cpp: 20 asan: off ubsan: off light_runtime: on @@ -48,6 +42,12 @@ jobs: asan: on ubsan: off light_runtime: off + - os: focal + compiler: g++-10 + cpp: 20 + asan: off + ubsan: off + light_runtime: on - os: jammy compiler: g++ cpp: 20 diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile new file mode 100644 index 0000000000..b9bae4f653 --- /dev/null +++ b/.github/workflows/Dockerfile @@ -0,0 +1,32 @@ +FROM debian:buster +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget lsb-release && \ + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ + echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-17 main" >> /etc/apt/sources.list && \ + echo "deb-src http://apt.llvm.org/buster/ llvm-toolchain-buster-17 main" >> /etc/apt/sources.list && \ + wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ + echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \ + wget -qO - https://debian.octopuce.fr/snapshots/sury-php/buster-latest/apt.gpg | apt-key add - && \ + echo "deb https://debian.octopuce.fr/snapshots/sury-php/buster-latest/ buster main" >> /etc/apt/sources.list && \ + TEMP_DEB="$(mktemp)" && \ + wget -O "$TEMP_DEB" 'https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb' && \ + dpkg -i "$TEMP_DEB" && \ + rm -f "$TEMP_DEB" && \ + echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + git cmake-data=3.18* cmake=3.18* make g++ clang-17 libclang-rt-17-dev gperf netcat \ + python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \ + pip3 install jsonschema && \ + apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ + libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ + postgresql postgresql-server-dev-all libnuma-dev composer libstdc++6 && \ + rm -rf /var/lib/apt/lists/* && \ + update-alternatives --set php /usr/bin/php7.4 + +RUN useradd -ms /bin/bash kitten diff --git a/.github/workflows/Dockerfile.buster b/.github/workflows/Dockerfile.buster index 432855b3c9..a3d62e1004 100644 --- a/.github/workflows/Dockerfile.buster +++ b/.github/workflows/Dockerfile.buster @@ -5,7 +5,10 @@ COPY tests/python/requirements.txt /tmp/ RUN apt-get update && \ apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget lsb-release && \ + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ + echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-17 main" >> /etc/apt/sources.list && \ + echo "deb-src http://apt.llvm.org/buster/ llvm-toolchain-buster-17 main" >> /etc/apt/sources.list && \ wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \ wget -qO - https://debian.octopuce.fr/snapshots/sury-php/buster-latest/apt.gpg | apt-key add - && \ @@ -19,12 +22,12 @@ RUN apt-get update && \ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 && \ apt-get update && \ apt-get install -y --no-install-recommends \ - git cmake-data=3.18* cmake=3.18* make g++-10 clang++ gperf netcat \ + git cmake-data=3.18* cmake=3.18* make g++ clang-17 libclang-rt-17-dev gperf netcat \ python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \ pip3 install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ - postgresql postgresql-server-dev-all libnuma-dev composer && \ + postgresql postgresql-server-dev-all libnuma-dev composer libstdc++6 libc++-6-dev && \ rm -rf /var/lib/apt/lists/* && \ update-alternatives --set php /usr/bin/php7.4 diff --git a/.github/workflows/Dockerfile.focal b/.github/workflows/Dockerfile.focal index 255e51a82d..e3f1cd848a 100644 --- a/.github/workflows/Dockerfile.focal +++ b/.github/workflows/Dockerfile.focal @@ -17,7 +17,7 @@ RUN apt-get update && \ python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ - postgresql postgresql-server-dev-all libnuma-dev composer unzip && \ + postgresql postgresql-server-dev-all libnuma-dev composer unzip libstdc++6 && \ rm -rf /var/lib/apt/lists/* ENV ASAN_OPTIONS=detect_leaks=0 From 318cfdae98ab325d6bf1c958e5202e72d495d212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 11:45:00 +0300 Subject: [PATCH 59/89] resolve From 7aaf62d068d857f8e22edd9ced4374abff5fa221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 10 Jul 2024 01:18:01 +0300 Subject: [PATCH 60/89] init --- .github/workflows/Build.yml | 34 +++++++++++++++++++++++++++++++--- .github/workflows/debian.yml | 11 +++++++++-- .github/workflows/ubuntu.yml | 23 ++++++++++++++++++++++- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index d3c8d31e5f..820f695dd6 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -23,23 +23,51 @@ jobs: cpp: 17 asan: off ubsan: off + light_runtime: off + - os: buster + compiler: g++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on - os: focal compiler: clang++ cpp: 17 asan: off ubsan: on + light_runtime: off + - os: focal + compiler: clang++ + cpp: 17 + asan: off + ubsan: on + light_runtime: on - os: focal compiler: g++-10 cpp: 20 asan: on ubsan: off + light_runtime: off + - os: focal + compiler: g++-10 + cpp: 20 + asan: on + ubsan: off + light_runtime: on - os: jammy compiler: g++ cpp: 20 asan: on ubsan: off - - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" + light_runtime: off + - os: jammy + compiler: g++ + cpp: 20 + asan: on + ubsan: off + light_runtime: on + + name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" steps: - uses: actions/checkout@v3 @@ -78,7 +106,7 @@ jobs: - name: Build all run: docker exec kphp-build-container-${{matrix.os}} bash -c - "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" + "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DCOMPILE_RUNTIME_LIGHT=${{matrix.light_runtime}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" - name: Run unit tests run: docker exec kphp-build-container-${{matrix.os}} bash -c diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 2d507d8110..1d365bd178 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -21,8 +21,15 @@ jobs: cpp: 17 asan: off ubsan: off + light_runtime: off + - os: buster + compiler: g++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" + name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" steps: - uses: actions/checkout@v3 @@ -59,7 +66,7 @@ jobs: - name: Build all run: docker exec -u kitten kphp-build-container-${{matrix.os}} bash -c - "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" + "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DCOMPILE_RUNTIME_LIGHT=${{matrix.light_runtime}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" - name: Run unit tests run: docker exec -u kitten kphp-build-container-${{matrix.os}} bash -c diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 49af16bd9f..0280b01a48 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -21,18 +21,39 @@ jobs: cpp: 17 asan: off ubsan: off + light_runtime: off + - os: jammy + compiler: g++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on - os: focal compiler: clang++ cpp: 17 asan: off ubsan: on + light_runtime: off + - os: focal + compiler: clang++ + cpp: 17 + asan: off + ubsan: on + light_runtime: on + - os: focal + compiler: g++-10 + cpp: 20 + asan: on + ubsan: off + light_runtime: off - os: focal compiler: g++-10 cpp: 20 asan: on ubsan: off + light_runtime: on - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" + name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" steps: - uses: actions/checkout@v3 From 66c040f252ce550d05e271561c45ee991be7f707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 10 Jul 2024 01:25:29 +0300 Subject: [PATCH 61/89] temporaly disable buster build die to sury issue --- .github/workflows/Build.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 820f695dd6..65b52c17b1 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -18,18 +18,18 @@ jobs: strategy: matrix: include: - - os: buster - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: off - - os: buster - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: on + # - os: buster + # compiler: g++ + # cpp: 17 + # asan: off + # ubsan: off + # light_runtime: off + # - os: buster + # compiler: g++ + # cpp: 17 + # asan: off + # ubsan: off + # light_runtime: on - os: focal compiler: clang++ cpp: 17 From ddc5272d0f9575370b65ea4024d7dd6a4478f9c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 10 Jul 2024 20:52:04 +0300 Subject: [PATCH 62/89] update --- .github/workflows/Build.yml | 44 +++++++++++------------------ .github/workflows/Dockerfile.buster | 31 -------------------- .github/workflows/debian.yml | 6 ++++ .github/workflows/ubuntu.yml | 23 +-------------- 4 files changed, 23 insertions(+), 81 deletions(-) delete mode 100644 .github/workflows/Dockerfile.buster diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 65b52c17b1..14c8941479 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -18,54 +18,42 @@ jobs: strategy: matrix: include: - # - os: buster - # compiler: g++ - # cpp: 17 - # asan: off - # ubsan: off - # light_runtime: off - # - os: buster - # compiler: g++ - # cpp: 17 - # asan: off - # ubsan: off - # light_runtime: on - - os: focal - compiler: clang++ + - os: buster + compiler: g++ cpp: 17 asan: off - ubsan: on + ubsan: off light_runtime: off - - os: focal + - os: buster + compiler: g++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on + - os: buster compiler: clang++ cpp: 17 asan: off - ubsan: on + ubsan: off light_runtime: on - os: focal - compiler: g++-10 - cpp: 20 - asan: on - ubsan: off + compiler: clang++ + cpp: 17 + asan: off + ubsan: on light_runtime: off - os: focal compiler: g++-10 cpp: 20 asan: on ubsan: off - light_runtime: on - - os: jammy - compiler: g++ - cpp: 20 - asan: on - ubsan: off light_runtime: off - os: jammy compiler: g++ cpp: 20 asan: on ubsan: off - light_runtime: on + light_runtime: off name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" diff --git a/.github/workflows/Dockerfile.buster b/.github/workflows/Dockerfile.buster deleted file mode 100644 index c19dd5ef5a..0000000000 --- a/.github/workflows/Dockerfile.buster +++ /dev/null @@ -1,31 +0,0 @@ -FROM debian:buster -ARG DEBIAN_FRONTEND=noninteractive - -COPY tests/python/requirements.txt /tmp/ - -RUN apt-get update && \ - apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget lsb-release && \ - echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ - wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ - echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \ - wget -qO - https://debian.octopuce.fr/snapshots/sury-php/buster-latest/apt.gpg | apt-key add - && \ - echo "deb https://debian.octopuce.fr/snapshots/sury-php/buster-latest/ buster main" >> /etc/apt/sources.list && \ - TEMP_DEB="$(mktemp)" && \ - wget -O "$TEMP_DEB" 'https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb' && \ - dpkg -i "$TEMP_DEB" && \ - rm -f "$TEMP_DEB" && \ - echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ - wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - git cmake-data=3.18* cmake=3.18* make g++ gperf netcat \ - python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \ - pip3 install -r /tmp/requirements.txt && \ - apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ - libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ - postgresql postgresql-server-dev-all libnuma-dev composer && \ - rm -rf /var/lib/apt/lists/* && \ - update-alternatives --set php /usr/bin/php7.4 - -RUN useradd -ms /bin/bash kitten diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 1d365bd178..d4c9130881 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -28,6 +28,12 @@ jobs: asan: off ubsan: off light_runtime: on + - os: buster + compiler: clang++ + cpp: 17 + asan: off + ubsan: off + light_runtime: on name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 0280b01a48..49af16bd9f 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -21,39 +21,18 @@ jobs: cpp: 17 asan: off ubsan: off - light_runtime: off - - os: jammy - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: on - os: focal compiler: clang++ cpp: 17 asan: off ubsan: on - light_runtime: off - - os: focal - compiler: clang++ - cpp: 17 - asan: off - ubsan: on - light_runtime: on - - os: focal - compiler: g++-10 - cpp: 20 - asan: on - ubsan: off - light_runtime: off - os: focal compiler: g++-10 cpp: 20 asan: on ubsan: off - light_runtime: on - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" + name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" steps: - uses: actions/checkout@v3 From 78bee1fc9d464604a3822cfb9cbfd712fac25a26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 11 Jul 2024 06:14:58 +0300 Subject: [PATCH 63/89] focal+gcc-10 and buster+clang --- .github/workflows/Build.yml | 16 +++++++-------- .github/workflows/Dockerfile | 32 ++++++++++++++++++++++++++++++ .github/workflows/Dockerfile.focal | 2 +- 3 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/Dockerfile diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 14c8941479..1a8a23f353 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -25,14 +25,8 @@ jobs: ubsan: off light_runtime: off - os: buster - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: on - - os: buster - compiler: clang++ - cpp: 17 + compiler: clang++-17 + cpp: 20 asan: off ubsan: off light_runtime: on @@ -48,6 +42,12 @@ jobs: asan: on ubsan: off light_runtime: off + - os: focal + compiler: g++-10 + cpp: 20 + asan: off + ubsan: off + light_runtime: on - os: jammy compiler: g++ cpp: 20 diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile new file mode 100644 index 0000000000..b9bae4f653 --- /dev/null +++ b/.github/workflows/Dockerfile @@ -0,0 +1,32 @@ +FROM debian:buster +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget lsb-release && \ + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ + echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-17 main" >> /etc/apt/sources.list && \ + echo "deb-src http://apt.llvm.org/buster/ llvm-toolchain-buster-17 main" >> /etc/apt/sources.list && \ + wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ + echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \ + wget -qO - https://debian.octopuce.fr/snapshots/sury-php/buster-latest/apt.gpg | apt-key add - && \ + echo "deb https://debian.octopuce.fr/snapshots/sury-php/buster-latest/ buster main" >> /etc/apt/sources.list && \ + TEMP_DEB="$(mktemp)" && \ + wget -O "$TEMP_DEB" 'https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb' && \ + dpkg -i "$TEMP_DEB" && \ + rm -f "$TEMP_DEB" && \ + echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + git cmake-data=3.18* cmake=3.18* make g++ clang-17 libclang-rt-17-dev gperf netcat \ + python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \ + pip3 install jsonschema && \ + apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ + libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ + postgresql postgresql-server-dev-all libnuma-dev composer libstdc++6 && \ + rm -rf /var/lib/apt/lists/* && \ + update-alternatives --set php /usr/bin/php7.4 + +RUN useradd -ms /bin/bash kitten diff --git a/.github/workflows/Dockerfile.focal b/.github/workflows/Dockerfile.focal index 255e51a82d..e3f1cd848a 100644 --- a/.github/workflows/Dockerfile.focal +++ b/.github/workflows/Dockerfile.focal @@ -17,7 +17,7 @@ RUN apt-get update && \ python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ - postgresql postgresql-server-dev-all libnuma-dev composer unzip && \ + postgresql postgresql-server-dev-all libnuma-dev composer unzip libstdc++6 && \ rm -rf /var/lib/apt/lists/* ENV ASAN_OPTIONS=detect_leaks=0 From a931aa615e53d764c04b3022a92c61fed34f7889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 11:45:00 +0300 Subject: [PATCH 64/89] resolve From 5f9c6b6de27504b31288e429d85044f6dfc2da2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 11:54:04 +0300 Subject: [PATCH 65/89] continue --- .github/workflows/{Dockerfile => Dockerfile.buster} | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) rename .github/workflows/{Dockerfile => Dockerfile.buster} (81%) diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile.buster similarity index 81% rename from .github/workflows/Dockerfile rename to .github/workflows/Dockerfile.buster index b9bae4f653..c19dd5ef5a 100644 --- a/.github/workflows/Dockerfile +++ b/.github/workflows/Dockerfile.buster @@ -1,12 +1,11 @@ FROM debian:buster ARG DEBIAN_FRONTEND=noninteractive +COPY tests/python/requirements.txt /tmp/ + RUN apt-get update && \ apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget lsb-release && \ - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ - echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-17 main" >> /etc/apt/sources.list && \ - echo "deb-src http://apt.llvm.org/buster/ llvm-toolchain-buster-17 main" >> /etc/apt/sources.list && \ wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \ wget -qO - https://debian.octopuce.fr/snapshots/sury-php/buster-latest/apt.gpg | apt-key add - && \ @@ -20,12 +19,12 @@ RUN apt-get update && \ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 && \ apt-get update && \ apt-get install -y --no-install-recommends \ - git cmake-data=3.18* cmake=3.18* make g++ clang-17 libclang-rt-17-dev gperf netcat \ + git cmake-data=3.18* cmake=3.18* make g++ gperf netcat \ python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \ - pip3 install jsonschema && \ + pip3 install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ - postgresql postgresql-server-dev-all libnuma-dev composer libstdc++6 && \ + postgresql postgresql-server-dev-all libnuma-dev composer && \ rm -rf /var/lib/apt/lists/* && \ update-alternatives --set php /usr/bin/php7.4 From 1f6928c2114334f00d213ceb603c0893cbc15879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 11:54:57 +0300 Subject: [PATCH 66/89] continue --- .github/workflows/Build.yml | 2 +- .github/workflows/debian.yml | 17 ++--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 2ac830c5af..d6b3eb4802 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -108,7 +108,7 @@ jobs: - name: Compile dummy PHP script if: ${{matrix.light_runtime}} run: docker exec kphp-build-container-${{matrix.os}} bash -c - "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --mode k2-component --cxx ${{matrix.compiler}} demo.php && kphp_out/cli --user kitten" + "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --mode k2-component --cxx ${{matrix.compiler}} demo.php" - name: Polyfills composer install run: docker exec kphp-build-container-${{matrix.os}} bash -c diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index d4c9130881..2d507d8110 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -21,21 +21,8 @@ jobs: cpp: 17 asan: off ubsan: off - light_runtime: off - - os: buster - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: on - - os: buster - compiler: clang++ - cpp: 17 - asan: off - ubsan: off - light_runtime: on - name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}" + name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" steps: - uses: actions/checkout@v3 @@ -72,7 +59,7 @@ jobs: - name: Build all run: docker exec -u kitten kphp-build-container-${{matrix.os}} bash -c - "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DCOMPILE_RUNTIME_LIGHT=${{matrix.light_runtime}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" + "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" - name: Run unit tests run: docker exec -u kitten kphp-build-container-${{matrix.os}} bash -c From 6500be48a39ccb933697080dde0dcd502ad3e3e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 12:33:06 +0300 Subject: [PATCH 67/89] g++-10 option when compiling script --- compiler/compiler-settings.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index 95f77c5735..71da41c13e 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -312,6 +312,7 @@ void CompilerSettings::init() { ss << " -std=c++17"; #elif __cplusplus <= 202002L ss << " -std=c++20"; + ss << " -fcoroutines"; #else #error unsupported __cplusplus value #endif From ea216c292ba88ef759ee1fc70cbd841a5f502b29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 12:44:05 +0300 Subject: [PATCH 68/89] update --- .github/workflows/Build.yml | 2 +- .github/workflows/Dockerfile.focal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index d6b3eb4802..1930066711 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -43,7 +43,7 @@ jobs: ubsan: off light_runtime: off - os: focal - compiler: g++-10 + compiler: g++-11 cpp: 20 asan: off ubsan: off diff --git a/.github/workflows/Dockerfile.focal b/.github/workflows/Dockerfile.focal index 9282166e69..15b1b822fb 100644 --- a/.github/workflows/Dockerfile.focal +++ b/.github/workflows/Dockerfile.focal @@ -15,7 +15,7 @@ RUN apt-get update && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ apt-get install -y --no-install-recommends \ - git cmake make clang g++ g++-10 clang-18 libclang-rt-18-dev gperf netcat \ + git cmake make clang g++ g++-10 g++-11 clang-18 libclang-rt-18-dev gperf netcat \ python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \ python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ From 2bcaf27037493f1ebf13e83c09610faaed78c578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 12:49:36 +0300 Subject: [PATCH 69/89] test --- .github/workflows/Build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 1930066711..bb3c266e31 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -67,7 +67,7 @@ jobs: path: 'kphp-polyfills' - name: Cache docker image - uses: actions/cache@v1 + uses: actions/cache@v3 id: docker-image-cache with: path: /tmp/docker-save @@ -76,7 +76,7 @@ jobs: - name: Build and save docker image if: steps.docker-image-cache.outputs.cache-hit != 'true' run: | - docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}} --cache-from=type=local,src=kphp-build-img-${{matrix.os}}-cache + docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} . -t kphp-build-img-${{matrix.os}} --cache-from=type=local,src=kphp-build-img-${{matrix.os}}-cache docker tag kphp-build-img-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache && mkdir -p /tmp/docker-save && docker save kphp-build-img-${{matrix.os}}-cache -o /tmp/docker-save/kphp-build-env-${{matrix.os}}.tar && ls -lh /tmp/docker-save - name: Load docker image from cache From 807e588baaed363f316c0c322a314918ddb19c6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 12:49:49 +0300 Subject: [PATCH 70/89] test2 --- .github/workflows/Build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index bb3c266e31..a80ec38f6d 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -76,7 +76,7 @@ jobs: - name: Build and save docker image if: steps.docker-image-cache.outputs.cache-hit != 'true' run: | - docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} . -t kphp-build-img-${{matrix.os}} --cache-from=type=local,src=kphp-build-img-${{matrix.os}}-cache + docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}} --cache-from=type=local,src=kphp-build-img-${{matrix.os}}-cache docker tag kphp-build-img-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache && mkdir -p /tmp/docker-save && docker save kphp-build-img-${{matrix.os}}-cache -o /tmp/docker-save/kphp-build-env-${{matrix.os}}.tar && ls -lh /tmp/docker-save - name: Load docker image from cache From d5482a67a7f04272149b9803dfab7df969ef5408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 12:56:30 +0300 Subject: [PATCH 71/89] test --- .github/workflows/Build.yml | 6 +++--- .github/workflows/Dockerfile.focal | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index a80ec38f6d..917d6e75c8 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -43,7 +43,7 @@ jobs: ubsan: off light_runtime: off - os: focal - compiler: g++-11 + compiler: g++-10 cpp: 20 asan: off ubsan: off @@ -67,7 +67,7 @@ jobs: path: 'kphp-polyfills' - name: Cache docker image - uses: actions/cache@v3 + uses: actions/cache@v1 id: docker-image-cache with: path: /tmp/docker-save @@ -76,7 +76,7 @@ jobs: - name: Build and save docker image if: steps.docker-image-cache.outputs.cache-hit != 'true' run: | - docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}} --cache-from=type=local,src=kphp-build-img-${{matrix.os}}-cache + docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} -t kphp-build-img-${{matrix.os}} --cache-from=type=local,src=kphp-build-img-${{matrix.os}}-cache docker tag kphp-build-img-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache && mkdir -p /tmp/docker-save && docker save kphp-build-img-${{matrix.os}}-cache -o /tmp/docker-save/kphp-build-env-${{matrix.os}}.tar && ls -lh /tmp/docker-save - name: Load docker image from cache diff --git a/.github/workflows/Dockerfile.focal b/.github/workflows/Dockerfile.focal index 15b1b822fb..9282166e69 100644 --- a/.github/workflows/Dockerfile.focal +++ b/.github/workflows/Dockerfile.focal @@ -15,7 +15,7 @@ RUN apt-get update && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ apt-get install -y --no-install-recommends \ - git cmake make clang g++ g++-10 g++-11 clang-18 libclang-rt-18-dev gperf netcat \ + git cmake make clang g++ g++-10 clang-18 libclang-rt-18-dev gperf netcat \ python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \ python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ From cb09c76a47401f212ab1187efcee4fb529825491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 16 Jul 2024 12:57:59 +0300 Subject: [PATCH 72/89] test --- .github/workflows/Build.yml | 54 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 917d6e75c8..fab9748e39 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -72,11 +72,11 @@ jobs: with: path: /tmp/docker-save key: docker-save-${{matrix.os}}-${{ hashFiles('.github/workflows/Dockerfile.*', 'tests/python/requirements.txt') }} - + - name: Build and save docker image if: steps.docker-image-cache.outputs.cache-hit != 'true' run: | - docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} -t kphp-build-img-${{matrix.os}} --cache-from=type=local,src=kphp-build-img-${{matrix.os}}-cache + docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}} --cache-from=kphp-build-img-${{matrix.os}}-cache docker tag kphp-build-img-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache && mkdir -p /tmp/docker-save && docker save kphp-build-img-${{matrix.os}}-cache -o /tmp/docker-save/kphp-build-env-${{matrix.os}}.tar && ls -lh /tmp/docker-save - name: Load docker image from cache @@ -85,7 +85,7 @@ jobs: - name: Start docker container run: | - docker run -dt --name kphp-build-container-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache + docker run -dt --name kphp-build-container-${{matrix.os}} kphp-build-img-${{matrix.os}} docker cp $GITHUB_WORKSPACE/. kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}} - name: Add git safe directory @@ -101,40 +101,40 @@ jobs: "make -C ${{env.kphp_build_dir}} -j$(nproc) test" - name: Compile dummy PHP script - if: ${{!matrix.light_runtime}} + if: ${{matrix.light_runtime}} == off run: docker exec kphp-build-container-${{matrix.os}} bash -c "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php && kphp_out/server -o --user kitten" - name: Compile dummy PHP script - if: ${{matrix.light_runtime}} + if: ${{matrix.light_runtime}} == on run: docker exec kphp-build-container-${{matrix.os}} bash -c - "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --mode k2-component --cxx ${{matrix.compiler}} demo.php" + "cd ${{env.kphp_build_dir}} && echo "${{matrix.light_runtime}}" && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --mode k2-component --cxx ${{matrix.compiler}} demo.php" - name: Polyfills composer install run: docker exec kphp-build-container-${{matrix.os}} bash -c "composer install -d ${{env.kphp_polyfills_dir}}" - - name: Run python tests - if: ${{!matrix.light_runtime}} - id: python_tests - continue-on-error: true - run: docker exec kphp-build-container-${{matrix.os}} bash -c - "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'" - - - name: Prepare python tests artifacts - if: ${{ (steps.python_tests.outcome == 'failure') && (!matrix.light_runtime) }} - run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} && - rm -rf ${{runner.temp}}/_tmp/*/working_dir - - - name: Upload python tests artifacts - uses: actions/upload-artifact@v3 - if: ${{ (steps.python_tests.outcome == 'failure') && (!matrix.light_runtime) }} - with: - path: ${{runner.temp}}/_tmp/ - - - name: Fail pipeline if python tests failed - if: ${{ (steps.python_tests.outcome == 'failure') && (!matrix.light_runtime) }} - run: exit 1 + # - name: Run python tests + # if: ${{!matrix.light_runtime}} + # id: python_tests + # continue-on-error: true + # run: docker exec kphp-build-container-${{matrix.os}} bash -c + # "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'" + + # - name: Prepare python tests artifacts + # if: ${{ (steps.python_tests.outcome == 'failure') && (!matrix.light_runtime) }} + # run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} && + # rm -rf ${{runner.temp}}/_tmp/*/working_dir + + # - name: Upload python tests artifacts + # uses: actions/upload-artifact@v3 + # if: ${{ (steps.python_tests.outcome == 'failure') && (!matrix.light_runtime) }} + # with: + # path: ${{runner.temp}}/_tmp/ + + # - name: Fail pipeline if python tests failed + # if: ${{ (steps.python_tests.outcome == 'failure') && (!matrix.light_runtime) }} + # run: exit 1 - name: Remove docker container run: docker rm -f kphp-build-container-${{matrix.os}} From 576ecc18f26ab2cdfe97a71ce895b04e2086b41a Mon Sep 17 00:00:00 2001 From: Vladislav Senin Date: Tue, 16 Jul 2024 19:22:15 +0300 Subject: [PATCH 73/89] fixed clang --asm-operand-widths warnings for aarch (#1031) --- common/crypto/aes256-aarch64.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/crypto/aes256-aarch64.cpp b/common/crypto/aes256-aarch64.cpp index 7e28cc67ea..4614aded21 100644 --- a/common/crypto/aes256-aarch64.cpp +++ b/common/crypto/aes256-aarch64.cpp @@ -97,7 +97,7 @@ void crypto_aarch64_aes256_cbc_encrypt(vk_aes_ctx_t *vk_ctx, const uint8_t *in, asm volatile("mov x9, %[iv] ;" // move IV address in x9 "mov x10, %[out] ;" // move out address in x10 - "mov x11, %[size] ;" // move size value in x11 + "mov x11, %x[size] ;" // move size value in x11 "mov x12, %[in] ;" // move plaintext address in x12 "mov x13, %[key] ;" // move key address in x13 "ld1 {v25.16b}, [x9] ;" // load IV to v0.16b @@ -167,7 +167,7 @@ void crypto_aarch64_aes256_cbc_decrypt(vk_aes_ctx_t *vk_ctx, const uint8_t *in, asm volatile("mov x9, %[iv] ;" // move IV address in x9 "mov x10, %[out] ;" // move out address in x10 - "mov x11, %[size] ;" // move size value in x11 + "mov x11, %x[size] ;" // move size value in x11 "mov x12, %[in] ;" // move ciphertext address in x12 "mov x13, %[key] ;" // move key address in x13 "ld1 {v25.16b}, [x9] ;" // load IV to v25.16b @@ -238,7 +238,7 @@ void crypto_aarch64_aes256_ige_encrypt(vk_aes_ctx_t *vk_ctx, const uint8_t *in, asm volatile("mov x9, %[iv] ;" // move IGE IV address in x9 "mov x10, %[out] ;" // move out address in x10 - "mov x11, %[size] ;" // move size value in x11 + "mov x11, %x[size] ;" // move size value in x11 "mov x12, %[in] ;" // move plaintext address in x12 "mov x13, %[key] ;" // move key address in x13 "ld1 {v25.16b}, [x9], #16 ;" // load IGE IV Y to v25.16b @@ -313,7 +313,7 @@ void crypto_aarch64_aes256_ige_decrypt(vk_aes_ctx_t *vk_ctx, const uint8_t *in, asm volatile("mov x9, %[iv] ;" // move IGE IV address in x9 "mov x10, %[out] ;" // move out address in x10 - "mov x11, %[size] ;" // move size value in x11 + "mov x11, %x[size] ;" // move size value in x11 "mov x12, %[in] ;" // move cyphertext address in x12 "mov x13, %[key] ;" // move key address in x13 "ld1 {v25.16b}, [x9], #16 ;" // load IGE IV Y to v25.16b @@ -435,8 +435,8 @@ static inline void crypto_aarch64_aes256_encrypt_single_block(vk_aes_ctx_t *vk_c } static inline void crypto_aarch64_aes256_encrypt_n_blocks(vk_aes_ctx_t *vk_ctx, const uint8_t *in, uint8_t *out, uint8_t iv[16], int n) { - asm volatile("mov x9, %[out] ;" // move out address in x9 - "mov x10, %[in] ;" // move plaintext address in x10 + asm volatile("mov x9, %[out] ;" // move out address in x9 + "mov x10, %x[in] ;" // move plaintext address in x10 "mov x11, %[key] ;" // move key address in x11 "mov x12, %[iv] ;" // move IV address in x12 "mov x13, %[n] ;" // move n value in x11 From 790db6e73cb86311d3d7335aee5aa71972431ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 17 Jul 2024 14:55:31 +0300 Subject: [PATCH 74/89] fcoroutins flag only with gcc, not with clang --- compiler/compiler-settings.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index 71da41c13e..65411373e1 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -312,7 +312,9 @@ void CompilerSettings::init() { ss << " -std=c++17"; #elif __cplusplus <= 202002L ss << " -std=c++20"; - ss << " -fcoroutines"; + #if (defined(__GNUC__) && !defined(__clang__)) + ss << " -fcoroutines"; + #endif #else #error unsupported __cplusplus value #endif From b611b47e3dec299f47381a1be34a30c2bfd9eb86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 17 Jul 2024 15:20:08 +0300 Subject: [PATCH 75/89] move to gcc-11 on focal --- .github/workflows/Build.yml | 2 +- .github/workflows/Dockerfile.focal | 3 ++- cmake/init-compilation-flags.cmake | 3 +-- compiler/compiler-settings.cpp | 4 +--- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index fab9748e39..ce5d3e1b26 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -43,7 +43,7 @@ jobs: ubsan: off light_runtime: off - os: focal - compiler: g++-10 + compiler: g++-11 cpp: 20 asan: off ubsan: off diff --git a/.github/workflows/Dockerfile.focal b/.github/workflows/Dockerfile.focal index 9282166e69..76755db75e 100644 --- a/.github/workflows/Dockerfile.focal +++ b/.github/workflows/Dockerfile.focal @@ -7,6 +7,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget pkg-config software-properties-common && \ wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \ wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + add-apt-repository ppa:ubuntu-toolchain-r/test && \ echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main" >> /etc/apt/sources.list && \ echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main" >> /etc/apt/sources.list && \ echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp focal main" >> /etc/apt/sources.list && \ @@ -15,7 +16,7 @@ RUN apt-get update && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ apt-get install -y --no-install-recommends \ - git cmake make clang g++ g++-10 clang-18 libclang-rt-18-dev gperf netcat \ + git cmake make clang g++ g++-10 g++-11 clang-18 libclang-rt-18-dev gperf netcat \ python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \ python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ diff --git a/cmake/init-compilation-flags.cmake b/cmake/init-compilation-flags.cmake index 0d18dff157..3d9e39fa78 100644 --- a/cmake/init-compilation-flags.cmake +++ b/cmake/init-compilation-flags.cmake @@ -10,8 +10,7 @@ if(CMAKE_CXX_COMPILER_ID MATCHES Clang) set(COMPILER_CLANG True) elseif(CMAKE_CXX_COMPILER_ID MATCHES GNU) if (COMPILE_RUNTIME_LIGHT) - check_compiler_version(gcc 10.1.0) - add_compile_options(-fcoroutines) + check_compiler_version(gcc 11.4.0) else() check_compiler_version(gcc 8.3.0) endif() diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index 65411373e1..1e211d14d9 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -312,9 +312,7 @@ void CompilerSettings::init() { ss << " -std=c++17"; #elif __cplusplus <= 202002L ss << " -std=c++20"; - #if (defined(__GNUC__) && !defined(__clang__)) - ss << " -fcoroutines"; - #endif + ss << "-Wno-unused-result -Wno-type-limits -Wno-attributes -Wno-ignored-attributes"; #else #error unsupported __cplusplus value #endif From f5a4edd6ee0e2b6621657c533db5cb15296d1397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 17 Jul 2024 15:46:39 +0300 Subject: [PATCH 76/89] fix conditions --- .github/workflows/Build.yml | 4 ++-- compiler/compiler-settings.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index ce5d3e1b26..10faf67f72 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -101,12 +101,12 @@ jobs: "make -C ${{env.kphp_build_dir}} -j$(nproc) test" - name: Compile dummy PHP script - if: ${{matrix.light_runtime}} == off + if: matrix.light_runtime == 'off' run: docker exec kphp-build-container-${{matrix.os}} bash -c "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php && kphp_out/server -o --user kitten" - name: Compile dummy PHP script - if: ${{matrix.light_runtime}} == on + if: matrix.light_runtime == 'on' run: docker exec kphp-build-container-${{matrix.os}} bash -c "cd ${{env.kphp_build_dir}} && echo "${{matrix.light_runtime}}" && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --mode k2-component --cxx ${{matrix.compiler}} demo.php" diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index 1e211d14d9..d19da1e18d 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -312,7 +312,7 @@ void CompilerSettings::init() { ss << " -std=c++17"; #elif __cplusplus <= 202002L ss << " -std=c++20"; - ss << "-Wno-unused-result -Wno-type-limits -Wno-attributes -Wno-ignored-attributes"; + ss << " -Wno-unused-result -Wno-type-limits -Wno-attributes -Wno-ignored-attributes"; #else #error unsupported __cplusplus value #endif From 8d194f1be6841c1409049fb72f183d0148a9d364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 17 Jul 2024 15:46:39 +0300 Subject: [PATCH 77/89] fix conditions --- .github/workflows/Build.yml | 4 ++-- cmake/init-compilation-flags.cmake | 1 + compiler/compiler-settings.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index ce5d3e1b26..10faf67f72 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -101,12 +101,12 @@ jobs: "make -C ${{env.kphp_build_dir}} -j$(nproc) test" - name: Compile dummy PHP script - if: ${{matrix.light_runtime}} == off + if: matrix.light_runtime == 'off' run: docker exec kphp-build-container-${{matrix.os}} bash -c "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php && kphp_out/server -o --user kitten" - name: Compile dummy PHP script - if: ${{matrix.light_runtime}} == on + if: matrix.light_runtime == 'on' run: docker exec kphp-build-container-${{matrix.os}} bash -c "cd ${{env.kphp_build_dir}} && echo "${{matrix.light_runtime}}" && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --mode k2-component --cxx ${{matrix.compiler}} demo.php" diff --git a/cmake/init-compilation-flags.cmake b/cmake/init-compilation-flags.cmake index 3d9e39fa78..0cbb1355f4 100644 --- a/cmake/init-compilation-flags.cmake +++ b/cmake/init-compilation-flags.cmake @@ -106,6 +106,7 @@ endif() add_compile_options(-Werror -Wall -Wextra -Wunused-function -Wfloat-conversion -Wno-sign-compare -Wuninitialized -Wno-redundant-move -Wno-missing-field-initializers) if(COMPILE_RUNTIME_LIGHT) + add_compile_options(-fPIC) add_compile_options(-Wno-unused-result -Wno-type-limits -Wno-attributes -Wno-ignored-attributes) add_compile_options(-Wno-vla-extension) endif() diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index 1e211d14d9..d19da1e18d 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -312,7 +312,7 @@ void CompilerSettings::init() { ss << " -std=c++17"; #elif __cplusplus <= 202002L ss << " -std=c++20"; - ss << "-Wno-unused-result -Wno-type-limits -Wno-attributes -Wno-ignored-attributes"; + ss << " -Wno-unused-result -Wno-type-limits -Wno-attributes -Wno-ignored-attributes"; #else #error unsupported __cplusplus value #endif From 6c495653ef48f8dcb040ca0f52d3161e2faeefb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 17 Jul 2024 16:37:59 +0300 Subject: [PATCH 78/89] try fix cache --- .github/workflows/Build.yml | 56 ++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 10faf67f72..0131dbe5cb 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -70,24 +70,24 @@ jobs: uses: actions/cache@v1 id: docker-image-cache with: - path: /tmp/docker-save + path: /tmp/docker-save-${{matrix.os}} key: docker-save-${{matrix.os}}-${{ hashFiles('.github/workflows/Dockerfile.*', 'tests/python/requirements.txt') }} - + - name: Build and save docker image if: steps.docker-image-cache.outputs.cache-hit != 'true' run: | - docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}} --cache-from=kphp-build-img-${{matrix.os}}-cache - docker tag kphp-build-img-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache && mkdir -p /tmp/docker-save && docker save kphp-build-img-${{matrix.os}}-cache -o /tmp/docker-save/kphp-build-env-${{matrix.os}}.tar && ls -lh /tmp/docker-save + docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}} --cache-from=type=local,src=kphp-build-img-${{matrix.os}}-cache + docker tag kphp-build-img-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache && mkdir -p /tmp/docker-save-${{matrix.os}} && docker save kphp-build-img-${{matrix.os}}-cache -o /tmp/docker-save-${{matrix.os}}/kphp-build-env-${{matrix.os}}.tar && ls -lh /tmp/docker-save-${{matrix.os}} - name: Load docker image from cache if: steps.docker-image-cache.outputs.cache-hit == 'true' - run: docker load --input /tmp/docker-save/kphp-build-env-${{matrix.os}}.tar + run: docker load --input /tmp/docker-save-${{matrix.os}}/kphp-build-env-${{matrix.os}}.tar - name: Start docker container run: | - docker run -dt --name kphp-build-container-${{matrix.os}} kphp-build-img-${{matrix.os}} + docker run -dt --name kphp-build-container-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache docker cp $GITHUB_WORKSPACE/. kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}} - + - name: Add git safe directory run: docker exec kphp-build-container-${{matrix.os}} bash -c "git config --global --add safe.directory ${{env.kphp_root_dir}}" @@ -114,27 +114,27 @@ jobs: run: docker exec kphp-build-container-${{matrix.os}} bash -c "composer install -d ${{env.kphp_polyfills_dir}}" - # - name: Run python tests - # if: ${{!matrix.light_runtime}} - # id: python_tests - # continue-on-error: true - # run: docker exec kphp-build-container-${{matrix.os}} bash -c - # "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'" - - # - name: Prepare python tests artifacts - # if: ${{ (steps.python_tests.outcome == 'failure') && (!matrix.light_runtime) }} - # run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} && - # rm -rf ${{runner.temp}}/_tmp/*/working_dir - - # - name: Upload python tests artifacts - # uses: actions/upload-artifact@v3 - # if: ${{ (steps.python_tests.outcome == 'failure') && (!matrix.light_runtime) }} - # with: - # path: ${{runner.temp}}/_tmp/ - - # - name: Fail pipeline if python tests failed - # if: ${{ (steps.python_tests.outcome == 'failure') && (!matrix.light_runtime) }} - # run: exit 1 + - name: Run python tests + if: matrix.light_runtime == 'off' + id: python_tests + continue-on-error: true + run: docker exec kphp-build-container-${{matrix.os}} bash -c + "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'" + + - name: Prepare python tests artifacts + if: ${{ (steps.python_tests.outcome == 'failure') && matrix.light_runtime == 'off' }} + run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} && + rm -rf ${{runner.temp}}/_tmp/*/working_dir + + - name: Upload python tests artifacts + uses: actions/upload-artifact@v3 + if: ${{ (steps.python_tests.outcome == 'failure') && matrix.light_runtime == 'off' }} + with: + path: ${{runner.temp}}/_tmp/ + + - name: Fail pipeline if python tests failed + if: ${{ (steps.python_tests.outcome == 'failure') && matrix.light_runtime == 'off' }} + run: exit 1 - name: Remove docker container run: docker rm -f kphp-build-container-${{matrix.os}} From bf88ac2078623e8add9f1fadf942709d5bbe4cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 17 Jul 2024 17:42:41 +0300 Subject: [PATCH 79/89] light runtime fpic --- cmake/init-compilation-flags.cmake | 1 - runtime-light/runtime-light.cmake | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmake/init-compilation-flags.cmake b/cmake/init-compilation-flags.cmake index 0cbb1355f4..3d9e39fa78 100644 --- a/cmake/init-compilation-flags.cmake +++ b/cmake/init-compilation-flags.cmake @@ -106,7 +106,6 @@ endif() add_compile_options(-Werror -Wall -Wextra -Wunused-function -Wfloat-conversion -Wno-sign-compare -Wuninitialized -Wno-redundant-move -Wno-missing-field-initializers) if(COMPILE_RUNTIME_LIGHT) - add_compile_options(-fPIC) add_compile_options(-Wno-unused-result -Wno-type-limits -Wno-attributes -Wno-ignored-attributes) add_compile_options(-Wno-vla-extension) endif() diff --git a/runtime-light/runtime-light.cmake b/runtime-light/runtime-light.cmake index c8d6d1c831..37db0aa5d7 100644 --- a/runtime-light/runtime-light.cmake +++ b/runtime-light/runtime-light.cmake @@ -30,7 +30,10 @@ set_target_properties(runtime-light PROPERTIES vk_add_library(kphp-light-runtime STATIC) target_link_libraries(kphp-light-runtime PUBLIC vk::light_common vk::runtime-light vk::runtime-core) -set_target_properties(kphp-light-runtime PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR}) +set_target_properties(kphp-light-runtime PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR} + POSITION_INDEPENDENT_CODE ON +) file(GLOB_RECURSE KPHP_RUNTIME_ALL_HEADERS RELATIVE ${BASE_DIR} From e8fd52f2fc4efaf568a195c8ec33a0120d72e14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 17 Jul 2024 17:46:46 +0300 Subject: [PATCH 80/89] cleanup --- .github/workflows/Build.yml | 9 +++++++-- .github/workflows/Dockerfile.focal | 4 ++-- runtime-core/runtime-core.cmake | 1 + runtime-light/runtime-light.cmake | 5 +---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 0131dbe5cb..517a800a32 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -67,7 +67,7 @@ jobs: path: 'kphp-polyfills' - name: Cache docker image - uses: actions/cache@v1 + uses: actions/cache@v3 id: docker-image-cache with: path: /tmp/docker-save-${{matrix.os}} @@ -87,11 +87,16 @@ jobs: run: | docker run -dt --name kphp-build-container-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache docker cp $GITHUB_WORKSPACE/. kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}} - + - name: Add git safe directory run: docker exec kphp-build-container-${{matrix.os}} bash -c "git config --global --add safe.directory ${{env.kphp_root_dir}}" + - name: Check formatting in light runtime folder + if: ${{ matrix.os == 'focal' && matrix.light_runtime == 'on' }} + run: docker exec kphp-build-container-${{matrix.os}} bash -c + "find ${{env.kphp_root_dir}}/runtime-light/ -iname '*.h' -o -iname '*.cpp' | xargs clang-format-18 -—dry-run -Werror" + - name: Build all run: docker exec kphp-build-container-${{matrix.os}} bash -c "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DCOMPILE_RUNTIME_LIGHT=${{matrix.light_runtime}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" diff --git a/.github/workflows/Dockerfile.focal b/.github/workflows/Dockerfile.focal index 76755db75e..8df13e0327 100644 --- a/.github/workflows/Dockerfile.focal +++ b/.github/workflows/Dockerfile.focal @@ -16,12 +16,12 @@ RUN apt-get update && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ apt-get install -y --no-install-recommends \ - git cmake make clang g++ g++-10 g++-11 clang-18 libclang-rt-18-dev gperf netcat \ + git cmake make clang g++ g++-10 g++-11 clang-18 libclang-rt-18-dev clang-format-18 gperf netcat \ python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \ python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \ - postgresql postgresql-server-dev-all libnuma-dev composer unzip libstdc++6 && \ + postgresql postgresql-server-dev-all libnuma-dev composer unzip && \ rm -rf /var/lib/apt/lists/* ENV ASAN_OPTIONS=detect_leaks=0 diff --git a/runtime-core/runtime-core.cmake b/runtime-core/runtime-core.cmake index be50be3f42..b43df8dcfd 100644 --- a/runtime-core/runtime-core.cmake +++ b/runtime-core/runtime-core.cmake @@ -23,3 +23,4 @@ set(KPHP_CORE_SRC ) vk_add_library(runtime-core OBJECT ${KPHP_CORE_SRC}) +set_property(TARGET runtime-core PROPERTY POSITION_INDEPENDENT_CODE ON) \ No newline at end of file diff --git a/runtime-light/runtime-light.cmake b/runtime-light/runtime-light.cmake index 37db0aa5d7..c8d6d1c831 100644 --- a/runtime-light/runtime-light.cmake +++ b/runtime-light/runtime-light.cmake @@ -30,10 +30,7 @@ set_target_properties(runtime-light PROPERTIES vk_add_library(kphp-light-runtime STATIC) target_link_libraries(kphp-light-runtime PUBLIC vk::light_common vk::runtime-light vk::runtime-core) -set_target_properties(kphp-light-runtime PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR} - POSITION_INDEPENDENT_CODE ON -) +set_target_properties(kphp-light-runtime PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR}) file(GLOB_RECURSE KPHP_RUNTIME_ALL_HEADERS RELATIVE ${BASE_DIR} From 32d1a2458b7d923d052ebe2183064822b3f51106 Mon Sep 17 00:00:00 2001 From: Vadim Sadokhov Date: Wed, 17 Jul 2024 18:46:13 +0300 Subject: [PATCH 81/89] format code --- runtime-light/header.h | 17 ++---- runtime-light/utils/json-functions.cpp | 66 ++++++++++-------------- runtime-light/utils/json-functions.h | 29 +++++------ runtime-light/utils/panic.h | 6 +-- runtime-light/utils/php_assert.h | 2 +- runtime-light/utils/to-array-processor.h | 4 +- 6 files changed, 53 insertions(+), 71 deletions(-) diff --git a/runtime-light/header.h b/runtime-light/header.h index 5bae9daf93..d513554a0f 100644 --- a/runtime-light/header.h +++ b/runtime-light/header.h @@ -107,8 +107,7 @@ struct PlatformCtx { * `stream_d` will be assigned `0`. * however `stream_d=0` itself is not an error marker */ - enum OpenStreamResult (*open)(size_t name_len, const char *name, - uint64_t *stream_d); + enum OpenStreamResult (*open)(size_t name_len, const char *name, uint64_t *stream_d); /* * If the write or read status is `Blocked` - then the platform ensures that * the component receives this `stream_d` via `take_update` when the status is @@ -118,8 +117,7 @@ struct PlatformCtx { * `new_status` will be assigned as * `{.read_status = 0, .write_status = 0, .please_shutdown = 0}`. */ - enum GetStatusResult (*get_stream_status)(uint64_t stream_d, - struct StreamStatus *new_status); + enum GetStatusResult (*get_stream_status)(uint64_t stream_d, struct StreamStatus *new_status); /* * Return processed bytes (written or read). * Guaranteed to return `0` if the stream is `Closed`, `Blocked` or @@ -190,8 +188,7 @@ struct PlatformCtx { * * `deadline` will be assigned `0` if `timer_d` invalid */ - enum TimerStatus (*get_timer_status)(uint64_t timer_d, - struct TimePoint *deadline); + enum TimerStatus (*get_timer_status)(uint64_t timer_d, struct TimePoint *deadline); /* * Return: `bool`. * If `True`: the update was successfully received. @@ -262,15 +259,11 @@ struct ImageInfo { }; // Every image should provide these symbols -enum PollStatus vk_k2_poll(const struct ImageState *image_state, - const struct PlatformCtx *pt_ctx, - struct ComponentState *component_ctx); +enum PollStatus vk_k2_poll(const struct ImageState *image_state, const struct PlatformCtx *pt_ctx, struct ComponentState *component_ctx); // platform_ctx without IO stuff (nullptr instead io-functions) // for now, returning nullptr will indicate error -struct ComponentState * -vk_k2_create_component_state(const struct ImageState *image_state, - const struct PlatformCtx *pt_ctx); +struct ComponentState *vk_k2_create_component_state(const struct ImageState *image_state, const struct PlatformCtx *pt_ctx); // platform_ctx without IO stuff (nullptr instead io-functions) // for now, returning nullptr will indicate error diff --git a/runtime-light/utils/json-functions.cpp b/runtime-light/utils/json-functions.cpp index 5fde8e0723..58d88c05a2 100644 --- a/runtime-light/utils/json-functions.cpp +++ b/runtime-light/utils/json-functions.cpp @@ -7,13 +7,13 @@ #include "common/algorithms/find.h" #include "runtime-light/component/component.h" // -//#include "runtime/string_functions.h" +// #include "runtime/string_functions.h" // note: json-functions.cpp is used for non-typed json implementation: for json_encode() and json_decode() // for classes, e.g. `JsonEncoder::encode(new A)`, see json-writer.cpp and from/to visitors namespace { -void json_append_one_char(unsigned int c, string_buffer & sb) noexcept { +void json_append_one_char(unsigned int c, string_buffer &sb) noexcept { sb.append_char('\\'); sb.append_char('u'); sb.append_char("0123456789abcdef"[c >> 12]); @@ -22,7 +22,7 @@ void json_append_one_char(unsigned int c, string_buffer & sb) noexcept { sb.append_char("0123456789abcdef"[c & 15]); } -bool json_append_char(unsigned int c, string_buffer & sb) noexcept { +bool json_append_char(unsigned int c, string_buffer &sb) noexcept { if (c < 0x10000) { if (0xD7FF < c && c < 0xE000) { return false; @@ -39,8 +39,7 @@ bool json_append_char(unsigned int c, string_buffer & sb) noexcept { return false; } - -bool do_json_encode_string_php(const JsonPath &json_path, const char *s, int len, int64_t options, string_buffer & sb) noexcept { +bool do_json_encode_string_php(const JsonPath &json_path, const char *s, int len, int64_t options, string_buffer &sb) noexcept { int begin_pos = sb.size(); if (options & JSON_UNESCAPED_UNICODE) { sb.reserve(2 * len + 2); @@ -178,7 +177,7 @@ string JsonPath::to_string() const { } unsigned num_parts = std::clamp(depth, 0U, static_cast(arr.size())); string result; - result.reserve_at_least((num_parts+1) * 8); + result.reserve_at_least((num_parts + 1) * 8); result.push_back('/'); for (unsigned i = 0; i < num_parts; i++) { const char *key = arr[i]; @@ -200,13 +199,12 @@ string JsonPath::to_string() const { namespace impl_ { -JsonEncoder::JsonEncoder(int64_t options, bool simple_encode, const char *json_obj_magic_key) noexcept: - options_(options), - simple_encode_(simple_encode), - json_obj_magic_key_(json_obj_magic_key) { -} +JsonEncoder::JsonEncoder(int64_t options, bool simple_encode, const char *json_obj_magic_key) noexcept + : options_(options) + , simple_encode_(simple_encode) + , json_obj_magic_key_(json_obj_magic_key) {} -bool JsonEncoder::encode(bool b, string_buffer & sb) noexcept { +bool JsonEncoder::encode(bool b, string_buffer &sb) noexcept { if (b) { sb.append("true", 4); } else { @@ -215,17 +213,17 @@ bool JsonEncoder::encode(bool b, string_buffer & sb) noexcept { return true; } -bool JsonEncoder::encode_null(string_buffer & sb) const noexcept { +bool JsonEncoder::encode_null(string_buffer &sb) const noexcept { sb.append("null", 4); return true; } -bool JsonEncoder::encode(int64_t i, string_buffer & sb) noexcept { +bool JsonEncoder::encode(int64_t i, string_buffer &sb) noexcept { sb << i; return true; } -bool JsonEncoder::encode(double d, string_buffer & sb) noexcept { +bool JsonEncoder::encode(double d, string_buffer &sb) noexcept { if (vk::any_of_equal(std::fpclassify(d), FP_INFINITE, FP_NAN)) { php_warning("%s: strange double %lf in function json_encode", json_path_.to_string().c_str(), d); if (options_ & JSON_PARTIAL_OUTPUT_ON_ERROR) { @@ -234,17 +232,17 @@ bool JsonEncoder::encode(double d, string_buffer & sb) noexcept { return false; } } else { - //todo:k2 implement f$number_format - sb << /*(simple_encode_ ? f$number_format(d, 6, string{"."}, string{}) : */ string{d}/*)*/; + // todo:k2 implement f$number_format + sb << /*(simple_encode_ ? f$number_format(d, 6, string{"."}, string{}) : */ string{d} /*)*/; } return true; } -bool JsonEncoder::encode(const string &s, string_buffer & sb) noexcept { +bool JsonEncoder::encode(const string &s, string_buffer &sb) noexcept { return do_json_encode_string_php(json_path_, s.c_str(), s.size(), options_, sb); } -bool JsonEncoder::encode(const mixed &v, string_buffer & sb) noexcept { +bool JsonEncoder::encode(const mixed &v, string_buffer &sb) noexcept { switch (v.get_type()) { case mixed::type::NUL: return encode_null(sb); @@ -280,29 +278,22 @@ bool do_json_decode(const char *s, int s_len, int &i, mixed &v, const char *json json_skip_blanks(s, i); switch (s[i]) { case 'n': - if (s[i + 1] == 'u' && - s[i + 2] == 'l' && - s[i + 3] == 'l') { + if (s[i + 1] == 'u' && s[i + 2] == 'l' && s[i + 3] == 'l') { i += 4; return true; } break; case 't': - if (s[i + 1] == 'r' && - s[i + 2] == 'u' && - s[i + 3] == 'e') { + if (s[i + 1] == 'r' && s[i + 2] == 'u' && s[i + 3] == 'e') { i += 4; - new(&v) mixed(true); + new (&v) mixed(true); return true; } break; case 'f': - if (s[i + 1] == 'a' && - s[i + 2] == 'l' && - s[i + 3] == 's' && - s[i + 4] == 'e') { + if (s[i + 1] == 'a' && s[i + 2] == 'l' && s[i + 3] == 's' && s[i + 4] == 'e') { i += 5; - new(&v) mixed(false); + new (&v) mixed(false); return true; } break; @@ -364,8 +355,7 @@ bool do_json_decode(const char *s, int s_len, int &i, mixed &v, const char *json } if (0xD7FF < num && num < 0xE000) { - if (s[i + 1] == '\\' && s[i + 2] == 'u' && - isxdigit(s[i + 3]) && isxdigit(s[i + 4]) && isxdigit(s[i + 5]) && isxdigit(s[i + 6])) { + if (s[i + 1] == '\\' && s[i + 2] == 'u' && isxdigit(s[i + 3]) && isxdigit(s[i + 4]) && isxdigit(s[i + 5]) && isxdigit(s[i + 6])) { i += 2; int u = 0; for (int t = 0; t < 4; t++) { @@ -419,7 +409,7 @@ bool do_json_decode(const char *s, int s_len, int &i, mixed &v, const char *json } value.shrink(l); - new(&v) mixed(value); + new (&v) mixed(value); i++; return true; } @@ -446,7 +436,7 @@ bool do_json_decode(const char *s, int s_len, int &i, mixed &v, const char *json i++; } - new(&v) mixed(res); + new (&v) mixed(res); return true; } case '{': { @@ -483,7 +473,7 @@ bool do_json_decode(const char *s, int s_len, int &i, mixed &v, const char *json res[string{json_obj_magic_key}] = true; } - new(&v) mixed(res); + new (&v) mixed(res); return true; } default: { @@ -495,7 +485,7 @@ bool do_json_decode(const char *s, int s_len, int &i, mixed &v, const char *json int64_t intval = 0; if (php_try_to_int(s + i, j - i, &intval)) { i = j; - new(&v) mixed(intval); + new (&v) mixed(intval); return true; } @@ -503,7 +493,7 @@ bool do_json_decode(const char *s, int s_len, int &i, mixed &v, const char *json double floatval = strtod(s + i, &end_ptr); if (end_ptr == s + j) { i = j; - new(&v) mixed(floatval); + new (&v) mixed(floatval); return true; } } diff --git a/runtime-light/utils/json-functions.h b/runtime-light/utils/json-functions.h index 15626a7e23..7805da79d9 100644 --- a/runtime-light/utils/json-functions.h +++ b/runtime-light/utils/json-functions.h @@ -9,7 +9,6 @@ #include "common/mixin/not_copyable.h" #include "runtime-core/runtime-core.h" - constexpr int64_t JSON_UNESCAPED_UNICODE = 1; constexpr int64_t JSON_FORCE_OBJECT = 16; constexpr int64_t JSON_PRETTY_PRINT = 128; // TODO: add actual support to untyped @@ -22,7 +21,7 @@ constexpr int64_t JSON_AVAILABLE_FLAGS_TYPED = JSON_PRETTY_PRINT | JSON_PRESERVE struct JsonPath { constexpr static int MAX_DEPTH = 8; - std::array arr; + std::array arr; unsigned depth = 0; void enter(const char *key) noexcept { @@ -47,31 +46,31 @@ class JsonEncoder : vk::not_copyable { public: JsonEncoder(int64_t options, bool simple_encode, const char *json_obj_magic_key = nullptr) noexcept; - //todo:k2 change static_SB everywhere to string_buffer arg - bool encode(bool b, string_buffer & sb) noexcept; - bool encode(int64_t i, string_buffer & sb) noexcept; - bool encode(const string &s, string_buffer & sb) noexcept; - bool encode(double d, string_buffer & sb) noexcept; - bool encode(const mixed &v, string_buffer & sb) noexcept; + // todo:k2 change static_SB everywhere to string_buffer arg + bool encode(bool b, string_buffer &sb) noexcept; + bool encode(int64_t i, string_buffer &sb) noexcept; + bool encode(const string &s, string_buffer &sb) noexcept; + bool encode(double d, string_buffer &sb) noexcept; + bool encode(const mixed &v, string_buffer &sb) noexcept; template - bool encode(const array &arr, string_buffer & sb) noexcept; + bool encode(const array &arr, string_buffer &sb) noexcept; template - bool encode(const Optional &opt, string_buffer & sb) noexcept; + bool encode(const Optional &opt, string_buffer &sb) noexcept; private: - bool encode_null(string_buffer & sb) const noexcept; + bool encode_null(string_buffer &sb) const noexcept; JsonPath json_path_; const int64_t options_{0}; - //todo:k2 use simple_encode + // todo:k2 use simple_encode [[maybe_unused]] const bool simple_encode_{false}; const char *json_obj_magic_key_{nullptr}; }; template -bool JsonEncoder::encode(const array &arr, string_buffer & sb) noexcept { +bool JsonEncoder::encode(const array &arr, string_buffer &sb) noexcept { bool is_vector = arr.is_vector(); const bool force_object = static_cast(JSON_FORCE_OBJECT & options_); if (!force_object && !is_vector && arr.is_pseudo_vector()) { @@ -142,7 +141,7 @@ bool JsonEncoder::encode(const array &arr, string_buffer & sb) noexcept { } template -bool JsonEncoder::encode(const Optional &opt, string_buffer & sb) noexcept { +bool JsonEncoder::encode(const Optional &opt, string_buffer &sb) noexcept { switch (opt.value_state()) { case OptionalState::has_value: return encode(opt.val(), sb); @@ -170,7 +169,7 @@ Optional f$json_encode(const T &v, int64_t options = 0, bool simple_enco return sb.c_str(); } -//todo:k2 implement string f$vk_json_encode_safe(const T &v, bool simple_encode = true) noexcept +// todo:k2 implement string f$vk_json_encode_safe(const T &v, bool simple_encode = true) noexcept template inline Optional f$vk_json_encode(const T &v) noexcept { diff --git a/runtime-light/utils/panic.h b/runtime-light/utils/panic.h index d11fbed624..7dfe5e956a 100644 --- a/runtime-light/utils/panic.h +++ b/runtime-light/utils/panic.h @@ -11,9 +11,9 @@ #include "runtime-light/utils/logs.h" inline void critical_error_handler() { - constexpr const char * message = "script panic"; - const PlatformCtx & ptx = *get_platform_context(); - ComponentState & ctx = *get_component_context(); + constexpr const char *message = "script panic"; + const PlatformCtx &ptx = *get_platform_context(); + ComponentState &ctx = *get_component_context(); ptx.log(Debug, strlen(message), message); if (ctx.not_finished()) { diff --git a/runtime-light/utils/php_assert.h b/runtime-light/utils/php_assert.h index 41d30d4f7b..81d534e5dd 100644 --- a/runtime-light/utils/php_assert.h +++ b/runtime-light/utils/php_assert.h @@ -7,7 +7,7 @@ #include #include -#include "common/wrappers/likely.h" #include "common/mixin/not_copyable.h" +#include "common/wrappers/likely.h" #include "runtime-core/utils/kphp-assert-core.h" diff --git a/runtime-light/utils/to-array-processor.h b/runtime-light/utils/to-array-processor.h index 19ac809395..47ff7ec4a2 100644 --- a/runtime-light/utils/to-array-processor.h +++ b/runtime-light/utils/to-array-processor.h @@ -86,7 +86,7 @@ class ToArrayVisitor { add_value(field_name, instance.is_null() ? mixed{} : f$to_array_debug(instance, with_class_names_)); } - template + template void process_impl(const char *field_name, const std::tuple &value) { ToArrayVisitor tuple_processor{with_class_names_}; tuple_processor.result_.reserve(sizeof...(Args), true); @@ -95,7 +95,7 @@ class ToArrayVisitor { add_value(field_name, std::move(tuple_processor).flush_result()); } - template + template void process_impl(const char *field_name, const shape, T...> &value) { ToArrayVisitor shape_processor{with_class_names_}; shape_processor.result_.reserve(sizeof...(Is), true); From 92ccc80db72e9e79c82ac111d286394086956c4e Mon Sep 17 00:00:00 2001 From: Vadim Sadokhov Date: Wed, 17 Jul 2024 18:53:04 +0300 Subject: [PATCH 82/89] remove less `-` in dry-run --- .github/workflows/Build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 517a800a32..7b12acad0c 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -95,7 +95,7 @@ jobs: - name: Check formatting in light runtime folder if: ${{ matrix.os == 'focal' && matrix.light_runtime == 'on' }} run: docker exec kphp-build-container-${{matrix.os}} bash -c - "find ${{env.kphp_root_dir}}/runtime-light/ -iname '*.h' -o -iname '*.cpp' | xargs clang-format-18 -—dry-run -Werror" + "find ${{env.kphp_root_dir}}/runtime-light/ -iname '*.h' -o -iname '*.cpp' | xargs clang-format-18 --dry-run -Werror" - name: Build all run: docker exec kphp-build-container-${{matrix.os}} bash -c From 4675969848496e6d2033f1e58f922514ba05fe93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 17 Jul 2024 19:37:16 +0300 Subject: [PATCH 83/89] fix macos gh build --- .github/workflows/Build.yml | 2 +- cmake/init-compilation-flags.cmake | 2 +- runtime-core/runtime-core.cmake | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 7b12acad0c..9828eb9005 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -170,7 +170,7 @@ jobs: brew update brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4 brew link --overwrite --force shivammathur/php/php@7.4 - /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install jsonschema install --break-system-packages jsonschema + /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install --break-system-packages jsonschema - name: Run cmake run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build diff --git a/cmake/init-compilation-flags.cmake b/cmake/init-compilation-flags.cmake index 3d9e39fa78..2729df9cd3 100644 --- a/cmake/init-compilation-flags.cmake +++ b/cmake/init-compilation-flags.cmake @@ -141,4 +141,4 @@ if(COMPILE_RUNTIME_LIGHT) message(FATAL_ERROR "Compiler or libstdc++ does not support coroutines") endif() file(REMOVE "${PROJECT_BINARY_DIR}/check_coroutine_include.cpp") -endif() \ No newline at end of file +endif() diff --git a/runtime-core/runtime-core.cmake b/runtime-core/runtime-core.cmake index b43df8dcfd..c2cd5ecdb4 100644 --- a/runtime-core/runtime-core.cmake +++ b/runtime-core/runtime-core.cmake @@ -23,4 +23,4 @@ set(KPHP_CORE_SRC ) vk_add_library(runtime-core OBJECT ${KPHP_CORE_SRC}) -set_property(TARGET runtime-core PROPERTY POSITION_INDEPENDENT_CODE ON) \ No newline at end of file +set_property(TARGET runtime-core PROPERTY POSITION_INDEPENDENT_CODE ON) From 937d928b48960b888a3d10640646563ab3449b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=83=D1=82=D1=8E=D0=BD=D1=8F=D0=BD=20=D0=90?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 17 Jul 2024 21:28:26 +0300 Subject: [PATCH 84/89] remove -Wno-unused-result flag --- cmake/init-compilation-flags.cmake | 2 +- common/dl-utils-lite.cpp | 9 +++++---- compiler/compiler-settings.cpp | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmake/init-compilation-flags.cmake b/cmake/init-compilation-flags.cmake index 2729df9cd3..268b0cda29 100644 --- a/cmake/init-compilation-flags.cmake +++ b/cmake/init-compilation-flags.cmake @@ -106,7 +106,7 @@ endif() add_compile_options(-Werror -Wall -Wextra -Wunused-function -Wfloat-conversion -Wno-sign-compare -Wuninitialized -Wno-redundant-move -Wno-missing-field-initializers) if(COMPILE_RUNTIME_LIGHT) - add_compile_options(-Wno-unused-result -Wno-type-limits -Wno-attributes -Wno-ignored-attributes) + add_compile_options(-Wno-type-limits -Wno-attributes -Wno-ignored-attributes) add_compile_options(-Wno-vla-extension) endif() diff --git a/common/dl-utils-lite.cpp b/common/dl-utils-lite.cpp index 8648274d6c..6bf9d35218 100644 --- a/common/dl-utils-lite.cpp +++ b/common/dl-utils-lite.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -49,9 +50,9 @@ double dl_time() { } void dl_print_backtrace(void **trace, int trace_size) { - write (2, "\n------- Stack Backtrace -------\n", 33); + std::ignore = write (2, "\n------- Stack Backtrace -------\n", 33); backtrace_symbols_fd (trace, trace_size, 2); - write (2, "-------------------------------\n", 32); + std::ignore = write (2, "-------------------------------\n", 32); } void dl_print_backtrace() { @@ -71,7 +72,7 @@ void dl_print_backtrace_gdb() { name_buf[res] = 0; int child_pid = fork(); if (child_pid < 0) { - write (2, "Can't fork() to run gdb\n", 24); + std::ignore = write (2, "Can't fork() to run gdb\n", 24); _exit (0); } if (!child_pid) { @@ -83,7 +84,7 @@ void dl_print_backtrace_gdb() { waitpid (child_pid, nullptr, 0); } } else { - write (2, "can't get name of executable file to pass to gdb\n", 49); + std::ignore = write (2, "can't get name of executable file to pass to gdb\n", 49); } } diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index d19da1e18d..abd519d677 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -312,7 +312,7 @@ void CompilerSettings::init() { ss << " -std=c++17"; #elif __cplusplus <= 202002L ss << " -std=c++20"; - ss << " -Wno-unused-result -Wno-type-limits -Wno-attributes -Wno-ignored-attributes"; + ss << " -Wno-type-limits -Wno-attributes -Wno-ignored-attributes"; #else #error unsupported __cplusplus value #endif From e690da8d6bcb0a8034676ddf70a36f81f5393438 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Wed, 17 Jul 2024 23:46:51 +0300 Subject: [PATCH 85/89] Delete sessions tests --- tests/cpp/runtime/sessions-test.cpp | 54 ----------------------------- 1 file changed, 54 deletions(-) delete mode 100644 tests/cpp/runtime/sessions-test.cpp diff --git a/tests/cpp/runtime/sessions-test.cpp b/tests/cpp/runtime/sessions-test.cpp deleted file mode 100644 index 2354389912..0000000000 --- a/tests/cpp/runtime/sessions-test.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include - -#include "runtime/files.h" -#include "runtime/sessions.h" - -TEST(sessions_test, test_session_id_with_invalid_id) { - const string dir_path = string(getenv("TMPDIR")).append("sessions/"); - const string id = string("\t12345678\\\r"); - ASSERT_FALSE(f$session_id(id).has_value()); - ASSERT_TRUE(f$session_start()); - ASSERT_NE(f$session_id().val(), id); - - const string full_path = string(dir_path).append(f$session_id().val()); - ASSERT_TRUE(f$file_exists(full_path)); - ASSERT_TRUE(f$session_abort()); -} - -TEST(sessions_test, test_session_id_with_valid_id) { - const string dir_path = string(getenv("TMPDIR")).append("sessions/"); - const string id = string("sess_668d4f818ca3b"); - ASSERT_FALSE(f$session_id(id).has_value()); - ASSERT_TRUE(f$session_start()); - ASSERT_EQ(f$session_id().val(), id); - - const string full_path = string(dir_path).append(id); - ASSERT_TRUE(f$file_exists(full_path)); - ASSERT_TRUE(f$session_abort()); -} - -TEST(sessions_test, test_session_start) { - ASSERT_TRUE(f$session_start()); - ASSERT_FALSE(f$session_start()); - ASSERT_TRUE(f$session_abort()); -} - -TEST(sessions_test, test_session_start_with_params) { - array predefined_consts = array(); - const string dir_path = string(getenv("TMPDIR")).append("example/"); - predefined_consts.emplace_value(string("save_path"), dir_path); - ASSERT_TRUE(f$session_start(predefined_consts)); - ASSERT_TRUE(f$file_exists(string(dir_path).append(f$session_id().val()))); - ASSERT_TRUE(f$session_abort()); -} - -TEST(sessions_test, test_session_status) { - const int SESSION_NONE = 1; - const int SESSION_ACTIVE = 2; - - ASSERT_EQ(f$session_status(), SESSION_NONE); - ASSERT_TRUE(f$session_start()); - ASSERT_EQ(f$session_status(), SESSION_ACTIVE); - ASSERT_TRUE(f$session_abort()); - ASSERT_EQ(f$session_status(), SESSION_NONE); -} \ No newline at end of file From 4216600c49b49320a39a28a5d6fed881dc79c3d0 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Thu, 18 Jul 2024 00:09:38 +0300 Subject: [PATCH 86/89] Delete sessions tests from cmake, hide buster in Build.yml --- .github/workflows/Build.yml | 12 ++++++------ tests/cpp/runtime/runtime-tests.cmake | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 7968850682..45a9fc98fa 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -21,12 +21,12 @@ jobs: strategy: matrix: include: - - os: buster - compiler: g++ - cpp: 17 - asan: off - ubsan: off - light_runtime: off + # - os: buster + # compiler: g++ + # cpp: 17 + # asan: off + # ubsan: off + # light_runtime: off - os: focal compiler: clang++ cpp: 17 diff --git a/tests/cpp/runtime/runtime-tests.cmake b/tests/cpp/runtime/runtime-tests.cmake index 1466b1a33d..731da98fe6 100644 --- a/tests/cpp/runtime/runtime-tests.cmake +++ b/tests/cpp/runtime/runtime-tests.cmake @@ -21,8 +21,7 @@ prepend(RUNTIME_TESTS_SOURCES ${BASE_DIR}/tests/cpp/runtime/ memory_resource/unsynchronized_pool_resource-test.cpp string-list-test.cpp string-test.cpp - zstd-test.cpp - sessions-test.cpp) + zstd-test.cpp) allow_deprecated_declarations_for_apple(${BASE_DIR}/tests/cpp/runtime/inter-process-mutex-test.cpp) vk_add_unittest(runtime "${RUNTIME_LIBS};${RUNTIME_LINK_TEST_LIBS}" ${RUNTIME_TESTS_SOURCES}) From 3b7c7d8ad90f0613d6ac7df86190946efbb84ff2 Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Thu, 18 Jul 2024 01:25:55 +0300 Subject: [PATCH 87/89] Add check on NULL in getenv --- runtime/sessions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index 5d6bc0ccbe..106f74d3f2 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -60,7 +60,7 @@ constexpr static auto C_HTTPONLY = "cookie_httponly"; // TO-DO: reconsider it const auto skeys = vk::to_array>({ {S_READ_CLOSE, false}, - {S_DIR, string(getenv("TMPDIR")).append("sessions/")}, + {S_DIR, string((getenv("TMPDIR") != NULL) ? getenv("TMPDIR") : "/").append("sessions/")}, {S_NAME, string("PHPSESSID")}, {S_LIFETIME, 1440}, {S_PROBABILITY, 1}, From 1427ecaad6baa0cb0e3c6d93d63ced2f1660086a Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Thu, 18 Jul 2024 01:26:32 +0300 Subject: [PATCH 88/89] Update Build.yml --- .github/workflows/Build.yml | 35 +++++++++++++++++++++-------------- docs/session_test.php | 14 +++++++++++++- runtime/sessions.cpp | 2 +- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 45a9fc98fa..1be4cffda0 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -21,12 +21,12 @@ jobs: strategy: matrix: include: - # - os: buster - # compiler: g++ - # cpp: 17 - # asan: off - # ubsan: off - # light_runtime: off + - os: buster + compiler: g++ + cpp: 17 + asan: off + ubsan: off + light_runtime: off - os: focal compiler: clang++ cpp: 17 @@ -63,6 +63,12 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Get stippets repo + uses: actions/checkout@v3 + with: + repository: 'VKCOM/kphp-snippets' + path: 'kphp-snippets' + - name: Get polyfills repo uses: actions/checkout@v3 with: @@ -146,15 +152,19 @@ jobs: - name: Copy the session_test file run: docker exec kphp-build-container-${{matrix.os}} bash -c - "su kitten -c 'cp ${{env.session_php_script}} ${{env.kphp_snippets_dir}}/JobWorkers/'" + "cp ${{env.session_php_script}} ${{env.kphp_snippets_dir}}/JobWorkers/" - name: Compile the session_test project run: docker exec kphp-build-container-${{matrix.os}} bash -c - "su kitten -c 'cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} session_test.php -I .. -M server'" + "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} session_test.php -I .. -M server" - name: Start the server run: docker exec kphp-build-container-${{matrix.os}} bash -c - "su kitten -c 'cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ./kphp_out/server -f 2 --http-port 8080 --job-workers-ratio 0.5 --sql-port 5555 &> log.txt &'" + "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ./kphp_out/server -f 2 --http-port 8080 --job-workers-ratio 0.5 --sql-port 5555 --user kitten &> log.txt &" + + - name: Install curl + run: docker exec kphp-build-container-${{matrix.os}} bash -c + "apt update && apt install curl" - name: Send a request to the server run: docker exec kphp-build-container-${{matrix.os}} bash -c @@ -162,11 +172,8 @@ jobs: - name: Read logs run: docker exec kphp-build-container-${{matrix.os}} bash -c - "su kitten -c 'cd ${{env.kphp_snippets_dir}}/JobWorkers/ && tail -f log.txt'" - - - name: Remove docker container - run: docker rm -f kphp-build-container-${{matrix.os}} - + "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && tail -f log.txt" + # build-macos: # runs-on: ${{matrix.os}}-12 # strategy: diff --git a/docs/session_test.php b/docs/session_test.php index b82e8cab7e..d17854197d 100644 --- a/docs/session_test.php +++ b/docs/session_test.php @@ -126,9 +126,13 @@ function handleHttpRequest() { $main_request = new MyRequest([], ["second_message" => "world"], $session_id, true); $main_job_id = \JobWorkers\JobLauncher::start($main_request, $timeout); - $new_request = new MyRequest([], ["third_message" => "buy"], $session_id, false); + $additional_to_main_request = new MyRequest([], ["third_message" => "buy"], $session_id, false); + $additional_to_main_job_id = \JobWorkers\JobLauncher::start($additional_to_main_request, $timeout); + + $new_request = new MyRequest([], ["new_message" => "hi"], false, false); $new_job_id = \JobWorkers\JobLauncher::start($new_request, $timeout); + $additional_to_main_response = wait($additional_to_main_job_id); $new_response = wait($new_job_id); $main_response = wait($main_job_id); @@ -147,6 +151,14 @@ function handleHttpRequest() { var_dump($new_response->session_array); echo "

"; } + + if ($additional_to_main_response instanceof MyResponse) { + echo "
Opened session:
"; + var_dump($additional_to_main_response->session_status); + var_dump($additional_to_main_response->session_id); + var_dump($additional_to_main_response->session_array); + echo "

"; + } } function handleKphpJobWorkerRequest() { diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index 106f74d3f2..f397e966dd 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -60,7 +60,7 @@ constexpr static auto C_HTTPONLY = "cookie_httponly"; // TO-DO: reconsider it const auto skeys = vk::to_array>({ {S_READ_CLOSE, false}, - {S_DIR, string((getenv("TMPDIR") != NULL) ? getenv("TMPDIR") : "/").append("sessions/")}, + {S_DIR, string((getenv("TMPDIR") != NULL) ? getenv("TMPDIR") : "/tmp/").append("sessions/")}, {S_NAME, string("PHPSESSID")}, {S_LIFETIME, 1440}, {S_PROBABILITY, 1}, From d7310094dfcd2de1d3221a5ece2847ea890de60c Mon Sep 17 00:00:00 2001 From: Marat Omarov Date: Sat, 20 Jul 2024 18:47:16 +0300 Subject: [PATCH 89/89] Change mode of session files, add logs for session_gc(), fix session_gc() --- .github/workflows/Build.yml | 9 +- docs/test_session_gc.php | 244 ++++++++++++++++++++++++++++++++++++ docs/test_wait_all.php | 228 +++++++++++++++++++++++++++++++++ runtime/sessions.cpp | 100 ++++++++++++++- 4 files changed, 572 insertions(+), 9 deletions(-) create mode 100644 docs/test_session_gc.php create mode 100644 docs/test_wait_all.php diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 1be4cffda0..137efc1335 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -11,7 +11,8 @@ env: kphp_root_dir: /home/kitten/kphp kphp_polyfills_dir: /home/kitten/kphp/kphp-polyfills kphp_build_dir: /home/kitten/kphp/build - session_php_script: /home/kitten/kphp/docs/session_test.php + test_session_php_script: /home/kitten/kphp/docs/test_session_gc.php + test_wait_all_php_script: /home/kitten/kphp/docs/test_wait_all.php kphp_snippets_dir: /home/kitten/kphp/kphp-snippets kphp_snippets_jobs_dir: /home/kitten/kphp/kphp-snippets/JobWorkers @@ -152,15 +153,15 @@ jobs: - name: Copy the session_test file run: docker exec kphp-build-container-${{matrix.os}} bash -c - "cp ${{env.session_php_script}} ${{env.kphp_snippets_dir}}/JobWorkers/" + "cp ${{env.test_wait_all_php_script}} ${{env.kphp_snippets_dir}}/JobWorkers/" - name: Compile the session_test project run: docker exec kphp-build-container-${{matrix.os}} bash -c - "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} session_test.php -I .. -M server" + "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} -I .. -M server test_wait_all.php" - name: Start the server run: docker exec kphp-build-container-${{matrix.os}} bash -c - "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ./kphp_out/server -f 2 --http-port 8080 --job-workers-ratio 0.5 --sql-port 5555 --user kitten &> log.txt &" + "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ./kphp_out/server -f 4 --http-port 8080 --job-workers-ratio 0.5 --sql-port 5555 --user kitten &> log.txt &" - name: Install curl run: docker exec kphp-build-container-${{matrix.os}} bash -c diff --git a/docs/test_session_gc.php b/docs/test_session_gc.php new file mode 100644 index 0000000000..99fd4e8737 --- /dev/null +++ b/docs/test_session_gc.php @@ -0,0 +1,244 @@ + */ + public $predefined_consts; + + /** @var mixed */ + public $session_array; + + /** @var bool */ + public $must_sleep; + + /** @var bool */ + public $must_call_gc; + + /** + * @param array $predefined_consts + * @param mixed $session_array + * @param false|string $id + * @param bool $must_sleep + * @param bool $must_call_gc + */ + public function __construct($predefined_consts, $session_array, $id, $must_sleep, $must_call_gc) { + $this->session_id = $id; + $this->predefined_consts = $predefined_consts; + $this->session_array = $session_array; + $this->must_sleep = $must_sleep; + $this->must_call_gc = $must_call_gc; + } + + function handleRequest(): ?\KphpJobWorkerResponse { + $response = new MyResponse(); + session_id($this->session_id); + + $response->start_time = microtime(true); + session_start($this->predefined_consts); + $response->session_status = (session_status() == 2); + $response->end_time = microtime(true); + + $response->session_id = session_id(); + if (!$response->session_status) { + return $response; + } + + $session_array_before = unserialize(session_encode()); + session_decode(serialize(array_merge($session_array_before, $this->session_array))); + // or: $_SESSION = array_merge($this->session_array, $_SESSION); + $response->session_array = unserialize(session_encode()); + // or: $response->session_array = $_SESSION; + + if ($this->must_sleep) { + usleep(4 * 100000); + } + + if ($this->must_call_gc) { + session_gc(); + } + + session_commit(); + + return $response; + } +} + +class MyResponse implements \KphpJobWorkerResponse { + /** @var float */ + public $start_time; + + /** @var float */ + public $end_time; + + /** @var bool */ + public $session_status; + + /** @var string|false */ + public $session_id; + + /** @var mixed */ + public $session_array; +} + + +if (PHP_SAPI !== 'cli' && isset($_SERVER["JOB_ID"])) { + handleKphpJobWorkerRequest(); +} else { + handleHttpRequest(); +} + +function handleHttpRequest() { + if (!\JobWorkers\JobLauncher::isEnabled()) { + echo "JOB WORKERS DISABLED at server start, use -f 2 --job-workers-ratio 0.5", "\n"; + return; + } + + $timeout = 4.5; + $to_write = ["first_message" => "hello"]; + $session_params = ["gc_maxlifetime" => 0]; + + $main_request = new MyRequest($session_params, $to_write, false, false, false); + $main_job_id = \JobWorkers\JobLauncher::start($main_request, $timeout); + + $main_response = wait($main_job_id); + $session_id = false; + + $is_response = ($main_response instanceof MyResponse); + var_dump($is_response); + + if ($main_response instanceof MyResponse) { + echo "\nOpened session:\n"; + var_dump($main_response->session_status); + var_dump($main_response->session_id); + var_dump($main_response->session_array); + $session_id = $main_response->session_id; + + var_dump($main_response->session_status); + var_dump($main_response->session_array == $to_write); + } else { + return; + } + + $session_params["gc_maxlifetime"] = 2000; + $add_request = new MyRequest($session_params, ["add_message" => "welcome"], false, false, false); + $add_job_id = \JobWorkers\JobLauncher::start($add_request, $timeout); + + $session_params["gc_maxlifetime"] = 1; + $main_request = new MyRequest($session_params, ["second_message" => "world"], $session_id, true, false); + $main_job_id = \JobWorkers\JobLauncher::start($main_request, $timeout); + $main_2_request = new MyRequest($session_params, ["third_message" => "buy"], $session_id, false, false); + $main_2_job_id = \JobWorkers\JobLauncher::start($main_2_request, $timeout); + + $new_request = new MyRequest($session_params, ["new_message" => "hi"], false, false, true); + $new_job_id = \JobWorkers\JobLauncher::start($new_request, $timeout); + + $add_response = wait($add_job_id); + $main_response = wait($main_job_id); + $main_2_response = wait($main_2_job_id); + $new_response = wait($new_job_id); + + $s_files = scandir($session_params["save_path"]); + if ($main_response instanceof MyResponse) { + echo "\nOpened session:\n"; + var_dump($main_response->session_status); + var_dump($main_response->session_id); + var_dump($main_response->session_array); + + $to_write["second_message"] = "world"; + var_dump($main_response->session_status); + var_dump($main_response->session_id == $session_id); + var_dump($main_response->session_array == $to_write); + var_dump(!in_array($main_response->session_id, $s_files)); + } + + if ($main_2_response instanceof MyResponse) { + echo "\nOpened session:\n"; + var_dump($main_2_response->session_status); + var_dump($main_2_response->session_id); + var_dump($main_2_response->session_array); + + $to_write["third_message"] = "buy"; + var_dump($main_2_response->session_status); + var_dump($main_2_response->session_id == $session_id); + var_dump($main_2_response->session_array == $to_write); + var_dump(!in_array($main_2_response->session_id, $s_files)); + } + + if ($add_response instanceof MyResponse) { + echo "\nOpened session:\n"; + var_dump($add_response->session_status); + var_dump($add_response->session_id); + var_dump($add_response->session_array); + + var_dump($add_response->session_status); + var_dump($add_response->session_id != $session_id); + var_dump($add_response->session_array == ["add_message" => "welcome"]); + var_dump(in_array($add_response->session_id, $s_files)); + } + + if ($new_response instanceof MyResponse) { + echo "\nOpened session:\n"; + var_dump($new_response->session_status); + var_dump($new_response->session_id); + var_dump($new_response->session_array); + + var_dump($new_response->session_status); + var_dump($new_response->session_id != $session_id); + var_dump($new_response->session_array == ["new_message" => "hi"]); + var_dump(in_array($new_response->session_id, $s_files)); + } +} + +function handleKphpJobWorkerRequest() { + $kphp_job_request = kphp_job_worker_fetch_request(); + if (!$kphp_job_request) { + warning("Couldn't fetch a job worker request"); + return; + } + + if ($kphp_job_request instanceof \JobWorkers\JobWorkerSimple) { + // simple jobs: they start, finish, and return the result + $kphp_job_request->beforeHandle(); + $response = $kphp_job_request->handleRequest(); + if ($response === null) { + warning("Job request handler returned null for " . get_class($kphp_job_request)); + return; + } + kphp_job_worker_store_response($response); + + } else if ($kphp_job_request instanceof \JobWorkers\JobWorkerManualRespond) { + // more complicated jobs: they start, send a result in the middle (here get get it) — and continue working + $kphp_job_request->beforeHandle(); + $kphp_job_request->handleRequest(); + if (!$kphp_job_request->wasResponded()) { + warning("Job request handler didn't call respondAndContinueExecution() manually " . get_class($kphp_job_request)); + } + + } else if ($kphp_job_request instanceof \JobWorkers\JobWorkerNoReply) { + // background jobs: they start and never send any result, just continue in the background and finish somewhen + $kphp_job_request->beforeHandle(); + $kphp_job_request->handleRequest(); + + } else { + warning("Got unexpected job request class: " . get_class($kphp_job_request)); + } +} diff --git a/docs/test_wait_all.php b/docs/test_wait_all.php new file mode 100644 index 0000000000..82bf4e48bb --- /dev/null +++ b/docs/test_wait_all.php @@ -0,0 +1,228 @@ + */ + public $predefined_consts; + + /** @var mixed */ + public $session_array; + + /** @var bool */ + public $must_sleep; + + /** @var bool */ + public $must_call_gc; + + /** + * @param array $predefined_consts + * @param mixed $session_array + * @param false|string $id + * @param string $title + * @param bool $must_sleep + * @param bool $must_call_gc + */ + public function __construct($predefined_consts, $session_array, $id, $title, $must_sleep, $must_call_gc) { + $this->session_id = $id; + $this->title = $title; + $this->predefined_consts = $predefined_consts; + $this->session_array = $session_array; + $this->must_sleep = $must_sleep; + $this->must_call_gc = $must_call_gc; + } + + function handleRequest(): ?\KphpJobWorkerResponse { + $response = new MyResponse(); + $response->title = $this->title; + session_id($this->session_id); + + $response->start_time = microtime(true); + session_start($this->predefined_consts); + $response->session_status = (session_status() == 2); + $response->end_time = microtime(true); + + $response->session_id = session_id(); + if (!$response->session_status) { + return $response; + } + + $session_array_before = unserialize(session_encode()); + session_decode(serialize(array_merge($session_array_before, $this->session_array))); + // or: $_SESSION = array_merge($this->session_array, $_SESSION); + $response->session_array = unserialize(session_encode()); + // or: $response->session_array = $_SESSION; + + if ($this->must_sleep) { + sleep(4); + } + + if ($this->must_call_gc) { + session_gc(); + } + + session_commit(); + + return $response; + } +} + +class MyResponse implements \KphpJobWorkerResponse { + /** @var float */ + public $start_time; + + /** @var float */ + public $end_time; + + /** @var bool */ + public $session_status; + + /** @var string|false */ + public $session_id; + + /** @var string */ + public $title; + + /** @var mixed */ + public $session_array; +} + + +if (PHP_SAPI !== 'cli' && isset($_SERVER["JOB_ID"])) { + handleKphpJobWorkerRequest(); +} else { + handleHttpRequest(); +} + +function handleHttpRequest() { + if (!\JobWorkers\JobLauncher::isEnabled()) { + echo "JOB WORKERS DISABLED at server start, use -f 2 --job-workers-ratio 0.5", "\n"; + return; + } + + $timeout = 4.5; + $to_write = ["first_message" => "hello"]; + $session_params = ["save_path" => "/tmp/sessions/", "gc_maxlifetime" => 2000]; + + $main_request = new MyRequest($session_params, $to_write, false, "main", false, false); + $main_job_id = \JobWorkers\JobLauncher::start($main_request, $timeout); + + $main_response = wait($main_job_id); + $session_id = false; + + $is_response = ($main_response instanceof MyResponse); + var_dump($is_response); + + if ($main_response instanceof MyResponse) { + echo "\nCreated main session:\n"; + var_dump($main_response->session_status); + var_dump($main_response->session_id); + var_dump($main_response->session_array); + $session_id = $main_response->session_id; + + var_dump($main_response->session_status); + var_dump($main_response->session_array == $to_write); + } else { + return; + } + + $futures_array = []; + + $session_params["gc_maxlifetime"] = 2000; + $add_request = new MyRequest($session_params, ["add_message" => "welcome"], false, "add", false, false); + $job_id = \JobWorkers\JobLauncher::start($add_request, $timeout); + if ($job_id !== false) { + $futures_array[] = $job_id; + } + + $session_params["gc_maxlifetime"] = 1; + $main_request = new MyRequest($session_params, ["second_message" => "world"], $session_id, "main", true, false); + $job_id = \JobWorkers\JobLauncher::start($main_request, $timeout); + if ($job_id !== false) { + $futures_array[] = $job_id; + } + + $main_2_request = new MyRequest($session_params, ["third_message" => "buy"], $session_id, "main_2", false, false); + $job_id = \JobWorkers\JobLauncher::start($main_2_request, $timeout); + if ($job_id !== false) { + $futures_array[] = $job_id; + } + + $new_request = new MyRequest($session_params, ["new_message" => "hi"], false, "new", false, true); + $job_id = \JobWorkers\JobLauncher::start($new_request, $timeout); + if ($job_id !== false) { + $futures_array[] = $job_id; + } + + $responses = wait_multi($futures_array); + $s_files = scandir($session_params["save_path"]); + var_dump($s_files); + + foreach ($responses as $response) { + if ($response instanceof MyResponse) { + var_dump($response->session_status); + var_dump($response->title); + var_dump($response->session_id); + var_dump($response->session_array); + } + } + + $conflicted_filename = $session_params["save_path"] . $session_id; + var_dump(file_get_contents($conflicted_filename)); +} + +function handleKphpJobWorkerRequest() { + $kphp_job_request = kphp_job_worker_fetch_request(); + if (!$kphp_job_request) { + warning("Couldn't fetch a job worker request"); + return; + } + + if ($kphp_job_request instanceof \JobWorkers\JobWorkerSimple) { + // simple jobs: they start, finish, and return the result + $kphp_job_request->beforeHandle(); + $response = $kphp_job_request->handleRequest(); + if ($response === null) { + warning("Job request handler returned null for " . get_class($kphp_job_request)); + return; + } + kphp_job_worker_store_response($response); + + } else if ($kphp_job_request instanceof \JobWorkers\JobWorkerManualRespond) { + // more complicated jobs: they start, send a result in the middle (here get get it) — and continue working + $kphp_job_request->beforeHandle(); + $kphp_job_request->handleRequest(); + if (!$kphp_job_request->wasResponded()) { + warning("Job request handler didn't call respondAndContinueExecution() manually " . get_class($kphp_job_request)); + } + + } else if ($kphp_job_request instanceof \JobWorkers\JobWorkerNoReply) { + // background jobs: they start and never send any result, just continue in the background and finish somewhen + $kphp_job_request->beforeHandle(); + $kphp_job_request->handleRequest(); + + } else { + warning("Got unexpected job request class: " . get_class($kphp_job_request)); + } +} diff --git a/runtime/sessions.cpp b/runtime/sessions.cpp index f397e966dd..23e2c3c539 100644 --- a/runtime/sessions.cpp +++ b/runtime/sessions.cpp @@ -12,6 +12,8 @@ #include "common/wrappers/to_array.h" #include "runtime/url.h" #include "runtime/math_functions.h" +// #include "common/smart_ptrs/unique_ptr_with_delete_function.h" +// #include "runtime/exec.h" namespace sessions { @@ -211,7 +213,7 @@ static bool session_open() { bool is_new = (!f$file_exists(get_sparam(S_PATH).to_string())) ? 1 : 0; fprintf(stderr, "[%d]\tOpening the session file %s\n", getpid(), get_sparam(S_PATH).to_string().c_str()); - set_sparam(S_FD, open_safe(get_sparam(S_PATH).to_string().c_str(), O_RDWR | O_CREAT, 0777)); + set_sparam(S_FD, open_safe(get_sparam(S_PATH).to_string().c_str(), O_RDWR | O_CREAT, 0666)); if (get_sparam(S_FD).to_int() < 0) { php_warning("Failed to open the file %s", get_sparam(S_PATH).to_string().c_str()); @@ -242,6 +244,9 @@ static bool session_open() { int ret_gc_lifetime = get_tag(get_sparam(S_PATH).to_string().c_str(), S_LIFETIME, NULL, 0); if (is_new or ret_ctime < 0) { // add the creation data to metadata of file + int is_session = 1; + set_tag(get_sparam(S_PATH).to_string().c_str(), "is_session", &is_session, sizeof(int)); + int ctime = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); set_tag(get_sparam(S_PATH).to_string().c_str(), S_CTIME, &ctime, sizeof(int)); } @@ -325,7 +330,7 @@ static bool session_write() { fprintf(stderr, "[%d]\t(rewinding) Closed the file\n", getpid()); fprintf(stderr, "[%d]\t(rewinding) Opening the file\n", getpid()); - set_sparam(S_FD, open_safe(get_sparam(S_PATH).to_string().c_str(), O_RDWR, 0777)); + set_sparam(S_FD, open_safe(get_sparam(S_PATH).to_string().c_str(), O_RDWR, 0666)); fprintf(stderr, "[%d]\t(rewinding) Opened the file\n", getpid()); lock.l_type = F_WRLCK; fprintf(stderr, "[%d]\t(rewinding) Locking the file\n", getpid()); @@ -398,6 +403,11 @@ static bool session_expired(const string &path) { return false; } +// for lsof tests +// static void pclose_wrapper(FILE *f) { +// pclose(f); +// } + static int session_gc(const bool &immediate = false) { double prob = f$lcg_value() * get_sparam(S_DIVISOR).to_float(); double s_prob = get_sparam(S_PROBABILITY).to_float(); @@ -411,6 +421,15 @@ static int session_gc(const bool &immediate = false) { return -1; } + // reset the fd before changing the session directory + close_safe(get_sparam(S_FD).to_int()); + + fprintf(stderr, "\n\tScan the session dir:\n\t"); + for (const auto &filename : s_list.as_array()) { + fprintf(stderr, "%s, ", filename.get_value().to_string().c_str()); + } + fprintf(stderr, "\n\n"); + struct flock lock; lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; @@ -421,29 +440,100 @@ static int session_gc(const bool &immediate = false) { int result = 0; for (auto s = s_list.as_array().begin(); s != s_list.as_array().end(); ++s) { string path = s.get_value().to_string(); - if (path == string(".") or path == string("..")) { + if (path[0] == '.') { continue; } + path = string(get_sparam(S_DIR).to_string()).append(path); if (path == get_sparam(S_PATH).to_string()) { continue; } + + { // filter session files from others + int is_session, ret_is_session = get_tag(path.c_str(), "is_session", &is_session, sizeof(int)); + if (ret_is_session < 0) { + continue; + } + } + + // { // lsof tests (non-working) + // // dl::CriticalSectionGuard heap_guard; + // // // string cmd = string("lsof +D ").append(get_sparam(S_DIR).to_string()); + // // string cmd = string("lsof"); + // // fprintf(stderr, "lsof cmd: %s\n", cmd.c_str()); + // // vk::unique_ptr_with_delete_function fp{popen(cmd.c_str(), "r")}; + // // if (fp != nullptr) { + // // std::array buff = {}; + // // string result; + // // const int fd = fileno(fp.get()); + // // std::size_t bytes_read = 0; + // // while ((bytes_read = read(fd, buff.data(), buff.size())) > 0) { + // // result.append(buff.data(), bytes_read); + // // } + + // // auto ret = pclose(fp.release()); + // // if (WIFEXITED(ret)) { + // // ret = WEXITSTATUS(ret); + // // } + + // // fprintf(stderr, "lsof result:\n%s\n", result.c_str()); + // // } + + // string cmd = string("lsof +D ").append(get_sparam(S_DIR).to_string()); + // // string cmd = string("lsof -t ").append(path).append(string(" 2>&1")); + // fprintf(stderr, "lsof cmd: %s\n", cmd.c_str()); + // mixed lsof_result; + // int64_t lsof_code; + // f$exec(cmd, lsof_result, lsof_code); + // fprintf(stderr, "lsof return code is %d\nlsof result:\n", static_cast(lsof_code)); + // if (lsof_result.is_array()) { + // for (const auto &it : lsof_result.as_array()) { + // fprintf(stderr, "%s\n", it.get_value().to_string().c_str()); + // } + // } + // } + fprintf(stderr, "\n\tOpening the session file %s\n", s.get_value().to_string().c_str()); int fd; - if ((fd = open(path.c_str(), O_RDWR, 0777)) < 0) { + if ((fd = open_safe(path.c_str(), O_RDWR, 0666)) < 0) { php_warning("Failed to open file on path: %s", path.c_str()); continue; } - + fprintf(stderr, "\tOpened the session file %s\n", s.get_value().to_string().c_str()); + + fprintf(stderr, "\tUnlocking the session file %s\n", s.get_value().to_string().c_str()); if (fcntl(fd, F_SETLK, &lock) < 0) { + fprintf(stderr, "\tsession file %s is opened, skip\n", s.get_value().to_string().c_str()); + close_safe(fd); continue; } + fprintf(stderr, "\tUnlocked the session file %s\n", s.get_value().to_string().c_str()); + close_safe(fd); if (session_expired(path)) { + fprintf(stderr, "\tThe session %s is expired, call unlink()\n\n", s.get_value().to_string().c_str()); f$unlink(path); ++result; } } + + lock.l_type = F_WRLCK; + lock.l_pid = getpid(); + set_sparam(S_FD, open_safe(get_sparam(S_PATH).to_string().c_str(), O_RDWR, 0666)); + if (get_sparam(S_FD).to_int() < 0) { + php_warning("Failed to reopen the file %s after session_gc()", get_sparam(S_PATH).to_string().c_str()); + session_abort(); + } else { + fcntl(get_sparam(S_FD).to_int(), F_SETLKW, &lock); + } + + fprintf(stderr, "\n\tScan the session dir after gc():\n\t"); + s_list = f$scandir(get_sparam(S_DIR).to_string()); + for (const auto &filename : s_list.as_array()) { + fprintf(stderr, "%s, ", filename.get_value().to_string().c_str()); + } + fprintf(stderr, "\n\n"); + return result; }