Test sessions

.github/CODEOWNERS

@@ -0,0 +1 @@
+.github/ @VKCOM/vk-sec

.github/workflows/Build.yml

@@ -11,6 +11,10 @@ env:
   kphp_root_dir: /home/kitten/kphp
   kphp_polyfills_dir: /home/kitten/kphp/kphp-polyfills
   kphp_build_dir: /home/kitten/kphp/build
+  test_session_php_script: /home/kitten/kphp/docs/test_session_gc.php
+  test_wait_all_php_script: /home/kitten/kphp/docs/test_wait_all.php
+  kphp_snippets_dir: /home/kitten/kphp/kphp-snippets
+  kphp_snippets_jobs_dir: /home/kitten/kphp/kphp-snippets/JobWorkers
 
 jobs:
   build-linux:
@@ -23,27 +27,49 @@ jobs:
             cpp: 17
             asan: off
             ubsan: off
+            light_runtime: off
           - os: focal
             compiler: clang++
             cpp: 17
             asan: off
             ubsan: on
+            light_runtime: off
           - os: focal
             compiler: g++-10
             cpp: 20
             asan: on
             ubsan: off
+            light_runtime: off
           - os: jammy
             compiler: g++
             cpp: 20
             asan: on
             ubsan: off
-
-    name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}"
+            light_runtime: off
+          # - os: focal
+          #   compiler: g++-11
+          #   cpp: 20
+          #   asan: off
+          #   ubsan: off
+          #   light_runtime: on
+          # - os: focal
+          #   compiler: clang++-18
+          #   cpp: 20
+          #   asan: off
+          #   ubsan: off
+          #   light_runtime: on
+
+    name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}/light_runtime=${{matrix.light_runtime}}"
 
     steps:
     - uses: actions/checkout@v3
 
+    - name: Get stippets repo
+      uses: actions/checkout@v3
+      with:
+        repository: 'VKCOM/kphp-snippets'
+        path: 'kphp-snippets'
+
     - name: Get polyfills repo
       uses: actions/checkout@v3
       with:
@@ -54,108 +80,141 @@ jobs:
       uses: actions/cache@v3
       id: docker-image-cache
       with:
-        path: kphp-build-env-${{matrix.os}}.tar
-        key: docker-image-cache-${{matrix.os}}-${{ hashFiles('.github/workflows/Dockerfile.*', 'tests/python/requirements.txt') }}
+        path: /tmp/docker-save-${{matrix.os}}
+        key: docker-save-${{matrix.os}}-${{ hashFiles('.github/workflows/Dockerfile.*', 'tests/python/requirements.txt') }}
 
     - name: Build and save docker image
       if: steps.docker-image-cache.outputs.cache-hit != 'true'
       run: |
-        docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}}
-        docker save --output kphp-build-env-${{matrix.os}}.tar kphp-build-img-${{matrix.os}}
+        docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}} $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}} --cache-from=type=local,src=kphp-build-img-${{matrix.os}}-cache
+        docker tag kphp-build-img-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache && mkdir -p /tmp/docker-save-${{matrix.os}} && docker save kphp-build-img-${{matrix.os}}-cache -o /tmp/docker-save-${{matrix.os}}/kphp-build-env-${{matrix.os}}.tar && ls -lh /tmp/docker-save-${{matrix.os}}
 
     - name: Load docker image from cache
       if: steps.docker-image-cache.outputs.cache-hit == 'true'
-      run: docker load --input kphp-build-env-${{matrix.os}}.tar
+      run: docker load --input /tmp/docker-save-${{matrix.os}}/kphp-build-env-${{matrix.os}}.tar
 
     - name: Start docker container
       run: |
-        docker run -dt --name kphp-build-container-${{matrix.os}} kphp-build-img-${{matrix.os}}
+        docker run -dt --name kphp-build-container-${{matrix.os}} kphp-build-img-${{matrix.os}}-cache
         docker cp $GITHUB_WORKSPACE/. kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}
 
     - name: Add git safe directory
       run: docker exec kphp-build-container-${{matrix.os}} bash -c
               "git config --global --add safe.directory ${{env.kphp_root_dir}}"
 
+    - name: Check formatting in light runtime folder
+      if: ${{ matrix.os == 'focal'  && matrix.light_runtime == 'on' }}
+      run:  docker exec kphp-build-container-${{matrix.os}} bash -c
+              "find ${{env.kphp_root_dir}}/runtime-light/ -iname '*.h' -o -iname '*.cpp' | xargs clang-format-18 --dry-run -Werror"
+
     - name: Build all
       run: docker exec kphp-build-container-${{matrix.os}} bash -c 
-              "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all"
-
-    - name: Run unit tests
-      run: docker exec kphp-build-container-${{matrix.os}} bash -c 
-              "make -C ${{env.kphp_build_dir}} -j$(nproc) test"
-
-    - name: Compile dummy PHP script
-      run: docker exec kphp-build-container-${{matrix.os}} bash -c 
-              "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php && kphp_out/server -o --user kitten"
-
-    - name: Polyfills composer install
+              "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DCOMPILE_RUNTIME_LIGHT=${{matrix.light_runtime}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all"
+
+    # - name: Run unit tests
+    #   run: docker exec kphp-build-container-${{matrix.os}} bash -c 
+    #           "make -C ${{env.kphp_build_dir}} -j$(nproc) test"
+
+    # - name: Compile dummy PHP script
+    #   if: matrix.light_runtime == 'off'
+    #   run: docker exec kphp-build-container-${{matrix.os}} bash -c 
+    #           "cd ${{env.kphp_build_dir}} && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php && kphp_out/server -o --user kitten"
+
+    # - name: Compile dummy PHP script
+    #   if: matrix.light_runtime == 'on'
+    #   run: docker exec kphp-build-container-${{matrix.os}} bash -c 
+    #           "cd ${{env.kphp_build_dir}} && echo "${{matrix.light_runtime}}" && echo 'hello world' > demo.php && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --mode k2-component --cxx ${{matrix.compiler}} demo.php"
+
+    # - name: Polyfills composer install
+    #   run: docker exec kphp-build-container-${{matrix.os}} bash -c
+    #           "composer install -d ${{env.kphp_polyfills_dir}}"
+
+    # - name: Run python tests
+    #   if: matrix.light_runtime == 'off'
+    #   id: python_tests
+    #   continue-on-error: true
+    #   run: docker exec kphp-build-container-${{matrix.os}} bash -c
+    #           "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'"
+
+    # - name: Prepare python tests artifacts
+    #   if: ${{ (steps.python_tests.outcome == 'failure') && matrix.light_runtime == 'off' }}
+    #   run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} &&
+    #        rm -rf ${{runner.temp}}/_tmp/*/working_dir
+
+    # - name: Upload python tests artifacts
+    #   uses: actions/upload-artifact@v3
+    #   if: ${{ (steps.python_tests.outcome == 'failure') && matrix.light_runtime == 'off' }}
+    #   with:
+    #     path: ${{runner.temp}}/_tmp/
+
+    # - name: Fail pipeline if python tests failed
+    #   if: ${{ (steps.python_tests.outcome == 'failure') && matrix.light_runtime == 'off' }}
+    #   run: exit 1
+
+    - name: Copy the session_test file
       run: docker exec kphp-build-container-${{matrix.os}} bash -c
-              "composer install -d ${{env.kphp_polyfills_dir}}"
+              "cp ${{env.test_wait_all_php_script}} ${{env.kphp_snippets_dir}}/JobWorkers/"
 
-    - name: Run python tests
-      id: python_tests
-      continue-on-error: true
+    - name: Compile the session_test project
       run: docker exec kphp-build-container-${{matrix.os}} bash -c
-              "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'"
-
-    - name: Prepare python tests artifacts
-      if: steps.python_tests.outcome == 'failure'
-      run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} &&
-           rm -rf ${{runner.temp}}/_tmp/*/working_dir
-
-    - name: Upload python tests artifacts
-      uses: actions/upload-artifact@v3
-      if: steps.python_tests.outcome == 'failure'
-      with:
-        path: ${{runner.temp}}/_tmp/
+              "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ${{env.kphp_root_dir}}/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} -I .. -M server test_wait_all.php"
 
-    - name: Fail pipeline if python tests failed
-      if: steps.python_tests.outcome == 'failure'
-      run: exit 1
+    - name: Start the server
+      run: docker exec kphp-build-container-${{matrix.os}} bash -c
+              "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && ./kphp_out/server -f 4 --http-port 8080 --job-workers-ratio 0.5 --sql-port 5555 --user kitten &> log.txt &"
 
-    - name: Remove docker container
-      run: docker rm -f kphp-build-container-${{matrix.os}}
+    - name: Install curl
+      run: docker exec kphp-build-container-${{matrix.os}} bash -c
+              "apt update && apt install curl"
 
-  build-macos:
-    runs-on: ${{matrix.os}}-12
-    strategy:
-      matrix:
-        include:
-          - os: macos
-            compiler: clang++
-            cpp: 17
+    - name: Send a request to the server
+      run: docker exec kphp-build-container-${{matrix.os}} bash -c
+              "curl 'http://localhost:8080'"
 
-    name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}"
+    - name: Read logs
+      run: docker exec kphp-build-container-${{matrix.os}} bash -c
+              "cd ${{env.kphp_snippets_dir}}/JobWorkers/ && tail -f log.txt"
+
+  # build-macos:
+  #   runs-on: ${{matrix.os}}-12
+  #   strategy:
+  #     matrix:
+  #       include:
+  #         - os: macos
+  #           compiler: clang++
+  #           cpp: 17
+
+  #   name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}"
 
-    steps:
-    - uses: actions/checkout@v3
-
-    # because of https://github.com/orgs/Homebrew/discussions/4612
-    - name: Check Environment
-      run: |
-        export HOMEBREW_NO_INSTALL_FROM_API=0
-        brew untap --force homebrew/cask
-
-    - name: Setup Environment
-      run: |
-        brew tap shivammathur/php
-        brew update
-        brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4
-        brew link --overwrite --force shivammathur/php/php@7.4
-        /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install jsonschema install --break-system-packages jsonschema
-
-    - name: Run cmake
-      run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build
-
-    - name: Build all
-      run: make -C ${{runner.workspace}}/build -j$(nproc) all
+  #   steps:
+  #   - uses: actions/checkout@v3
+
+  #   # because of https://github.com/orgs/Homebrew/discussions/4612
+  #   - name: Check Environment
+  #     run: |
+  #       export HOMEBREW_NO_INSTALL_FROM_API=0
+  #       brew untap --force homebrew/cask
+
+  #   - name: Setup Environment
+  #     run: |
+  #       brew tap shivammathur/php
+  #       brew update
+  #       brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4
+  #       brew link --overwrite --force shivammathur/php/php@7.4
+  #       /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install --break-system-packages jsonschema
+
+  #   - name: Run cmake
+  #     run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build
+
+  #   - name: Build all
+  #     run: make -C ${{runner.workspace}}/build -j$(nproc) all
 
-    - name: Run unit tests
-      run: make -C ${{runner.workspace}}/build -j$(nproc) test
-
-    - name: Compile dummy PHP script
-      working-directory: ${{runner.workspace}}/build
-      run: |
-        echo 'hello world' > demo.php 
-        $GITHUB_WORKSPACE/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php
-        kphp_out/server -o
+  #   - name: Run unit tests
+  #     run: make -C ${{runner.workspace}}/build -j$(nproc) test
+
+  #   - name: Compile dummy PHP script
+  #     working-directory: ${{runner.workspace}}/build
+  #     run: |
+  #       echo 'hello world' > demo.php 
+  #       $GITHUB_WORKSPACE/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php
+  #       kphp_out/server -o

.github/workflows/Dockerfile.buster

@@ -5,11 +5,11 @@ COPY tests/python/requirements.txt /tmp/
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget lsb-release && \
-    echo "deb https://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ 
+    echo "deb https://archive.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \
     wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \
     echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp buster main" >> /etc/apt/sources.list && \
-    wget -qO - https://packages.sury.org/php/apt.gpg | apt-key add - && \
-    echo "deb https://packages.sury.org/php/ buster main" >> /etc/apt/sources.list.d/php.list && \
+    wget -qO - https://debian.octopuce.fr/snapshots/sury-php/buster-latest/apt.gpg | apt-key add - && \
+    echo "deb https://debian.octopuce.fr/snapshots/sury-php/buster-latest/ buster main" >> /etc/apt/sources.list && \
     TEMP_DEB="$(mktemp)" && \
     wget -O "$TEMP_DEB" 'https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb' && \
     dpkg -i "$TEMP_DEB" && \
@@ -20,7 +20,7 @@ RUN apt-get update && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
       git cmake-data=3.18* cmake=3.18* make g++ gperf netcat \
-      python3.7 python3-dev libpython3-dev python3-pip python3-setuptools mysql-server libmysqlclient-dev && \
+      python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \
     pip3 install -r /tmp/requirements.txt && \
     apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
       libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \

.github/workflows/Dockerfile.focal

@@ -6,13 +6,17 @@ COPY tests/python/requirements.txt /tmp/
 RUN apt-get update && \
     apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget pkg-config software-properties-common && \
     wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \
+    wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    add-apt-repository ppa:ubuntu-toolchain-r/test && \
+    echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main" >> /etc/apt/sources.list && \
+    echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main" >> /etc/apt/sources.list && \
     echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp focal main" >> /etc/apt/sources.list && \
     echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \
     wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
     add-apt-repository ppa:deadsnakes/ppa && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
-      git cmake make clang g++ g++-10 gperf netcat \
+      git cmake make clang g++ g++-10 g++-11 clang-18 libclang-rt-18-dev clang-format-18 gperf netcat \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \
     python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
     apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \

.gitignore

@@ -8,6 +8,7 @@
 .DS_Store
 
 .idea
+.vscode
 build
 *.iml
 /cmake-build-*