diff --git a/.github/workflows/build-test-release.yaml b/.github/workflows/build-test-release.yaml index 230d5daf..9729d109 100644 --- a/.github/workflows/build-test-release.yaml +++ b/.github/workflows/build-test-release.yaml @@ -19,7 +19,7 @@ jobs: compliance-copyrights: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: apache/skywalking-eyes@v0.6.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -27,7 +27,7 @@ jobs: pre-commit: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: "3.7" @@ -47,7 +47,7 @@ jobs: client-id: ${{ secrets.GH_APP_CLIENT_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: false persist-credentials: false @@ -72,7 +72,7 @@ jobs: client-id: ${{ secrets.GH_APP_CLIENT_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: splunk/addonfactory-update-semver@v1 env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} diff --git a/.github/workflows/reusable-build-test-release.yml b/.github/workflows/reusable-build-test-release.yml index 0007702a..f0fae411 100644 --- a/.github/workflows/reusable-build-test-release.yml +++ b/.github/workflows/reusable-build-test-release.yml @@ -39,7 +39,7 @@ on: required: false description: "branch for k8s manifests to run the tests on" type: string - default: "v4.2" + default: "v4.3" scripted-inputs-os-list: required: false description: "list of OS used for scripted input tests" @@ -77,11 +77,6 @@ on: description: "Version of the GS scorecard" type: string default: "0.3" - fossa-ignore-vulnerabilities: - required: false - description: 'JSON array of "package@version" strings identifying known false-positive FOSSA vulnerability issues to exclude from the fossa-vulnerability-test job. Example: ["urllib3@1.26.19"]' - type: string - default: "[]" secrets: GH_TOKEN_ADMIN: description: Github admin token @@ -148,7 +143,7 @@ concurrency: cancel-in-progress: true env: PYTHON_VERSION: ${{ inputs.python-version }} - POETRY_VERSION: "2.1.4" + POETRY_VERSION: "2.2.1" POETRY_EXPORT_PLUGIN_VERSION: "1.9.0" GS_IMAGE_VERSION: ${{ inputs.gs-image-version }} GS_VERSION: ${{ inputs.gs-version }} @@ -157,7 +152,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event.inputs.custom-version != '' }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Validate custom version run: | if [[ ! ${{ github.event.inputs.custom-version }} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -174,7 +169,7 @@ jobs: check-splunktafunctionaltests-exists: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - run: | if grep -q 'splunktafunctionaltests' poetry.lock || grep -q 'splunktafunctionaltests' dev_deps/requirements_dev.txt 2>/dev/null; then echo "::warning title=\"splunktafunctionaltests\" should NOT be used for modinput tests::For more details, please see https://splunk.slack.com/archives/C081JT7R69Z/p1754662758743839." @@ -189,7 +184,7 @@ jobs: docs-only: ${{ steps.check.outputs.docs-only }} steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Fetch all refs run: git fetch --prune --unshallow @@ -249,7 +244,7 @@ jobs: fi - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Determine Test Execution Flags id: determine-test-execution-flags @@ -297,15 +292,15 @@ jobs: case "${{ github.event_name }}" in "pull_request") labels=$(echo '${{ toJSON(github.event.pull_request.labels) }}' | jq -r '.[] | .name') - if ${{ github.base_ref == 'main' }} && ${{ contains(github.event.pull_request.labels.*.name, 'use_labels') }}; then + if ${{ (github.base_ref == 'main' || startsWith(github.base_ref, 'release/')) && contains(github.event.pull_request.labels.*.name, 'use_labels') }}; then for test_type in "${TESTS_TO_CONSIDER_FOR_EXECUTION[@]}"; do if [[ "$labels" =~ $test_type ]]; then EXECUTION_FLAGS["$test_type"]="true" fi done - elif ${{ github.base_ref == 'main' }} || ${{ contains(github.event.pull_request.labels.*.name, 'execute_all_tests') }}; then + elif ${{ github.base_ref == 'main' || startsWith(github.base_ref, 'release/') || contains(github.event.pull_request.labels.*.name, 'execute_all_tests') }}; then for test_type in "${TESTS_TO_CONSIDER_FOR_EXECUTION[@]}"; do - # Exclude upgrade tests on PRs to main + # Exclude upgrade tests on PRs to main or release branches if [[ "$test_type" != "execute_upgrade" ]]; then EXECUTION_FLAGS["$test_type"]="true" fi @@ -467,6 +462,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix_Splunk: ${{ steps.determine_splunk.outputs.matrixSplunk }} + matrix_splunkModinput: ${{ steps.determine_splunk.outputs.matrixSplunkModinput }} matrix_supportedSplunk: ${{ steps.matrix.outputs.supportedSplunk }} matrix_latestSplunk: ${{ steps.matrix.outputs.latestSplunk }} matrix_supportedSC4S: ${{ steps.matrix.outputs.supportedSC4S }} @@ -476,12 +472,12 @@ jobs: contents: write packages: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: false persist-credentials: false - id: matrix - uses: splunk/addonfactory-test-matrix-action@v3.2 + uses: splunk/addonfactory-test-matrix-action@v3.3 with: features: PYTHON39 - id: determine_splunk @@ -493,7 +489,9 @@ jobs: else if [[ "${{ github.base_ref }}" == "main" || "${{ github.ref_name }}" == "main" ]] || \ - [[ "${{ github.ref_name }}" == "develop" && "${{ github.event_name }}" == "push" ]]; then + [[ "${{ github.ref_name }}" == "develop" && "${{ github.event_name }}" == "push" ]] || \ + [[ "${{ startsWith(github.base_ref, 'release/') }}" == "true" ]] || \ + [[ "${{ startsWith(github.ref_name, 'release/') }}" == "true" && "${{ github.event_name }}" == "push" && "${{ inputs.execute-tests-on-push-to-release }}" == "true" ]]; then wfe_run_on_splunk_latest="false" else wfe_run_on_splunk_latest="true" @@ -501,8 +499,10 @@ jobs: fi if [[ "$wfe_run_on_splunk_latest" == "true" ]]; then echo "matrixSplunk=${{ toJson(steps.matrix.outputs.latestSplunk) }}" >> "$GITHUB_OUTPUT" + echo "matrixSplunkModinput=${{ toJson(steps.matrix.outputs.latestSplunk) }}" >> "$GITHUB_OUTPUT" else echo "matrixSplunk=${{ toJson(steps.matrix.outputs.supportedSplunk) }}" >> "$GITHUB_OUTPUT" + echo "matrixSplunkModinput=${{ toJson(steps.matrix.outputs.supportedSplunkModinput) }}" >> "$GITHUB_OUTPUT" fi - name: job summary @@ -516,7 +516,7 @@ jobs: outputs: report-url: ${{ steps.fossa-scan.outputs.FOSSA_REPORT_URL }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: run fossa analyze and create report id: fossa-scan run: | @@ -537,13 +537,13 @@ jobs: env: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} - name: upload THIRDPARTY file - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: THIRDPARTY path: /tmp/THIRDPARTY - name: upload fossa test output if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: fossa-test-output path: fossa-test-output.txt @@ -562,7 +562,7 @@ jobs: active-issues: ${{ steps.fossa-license-test.outputs.active-issues }} steps: - name: download fossa test output - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: fossa-test-output - name: run fossa license test @@ -620,8 +620,9 @@ jobs: outputs: active-issues: ${{ steps.fossa-vulnerability-test.outputs.active-issues }} steps: + - uses: actions/checkout@v7 - name: download fossa test output - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: fossa-test-output - name: run fossa vulnerability test @@ -629,12 +630,14 @@ jobs: shell: bash env: FOSSA_REPORT_URL: ${{ needs.fossa-scan.outputs.report-url }} - FOSSA_IGNORE_VULNERABILITIES: ${{ inputs.fossa-ignore-vulnerabilities }} run: | set -euo pipefail header=$(awk '/^ *Using project name:|^ *Using revision:/{print}' fossa-test-output.txt || true) raw_section=$(awk '/^ *COMPLIANCE ISSUES/{if(found) exit} /^ *SECURITY ISSUES/{found=1} found{print}' fossa-test-output.txt || true) - ignore_packages=$(echo "$FOSSA_IGNORE_VULNERABILITIES" | jq -r '.[]' 2>/dev/null || true) + ignore_packages="" + if [[ -f "package/lib/exclude.txt" ]]; then + ignore_packages=$(< package/lib/exclude.txt) + fi section=$(echo "$raw_section" | awk -v ignore="$ignore_packages" ' BEGIN { skip = 0 @@ -643,11 +646,11 @@ jobs: } /⚑/ { skip = 0 - if (match($0, /on [^ ]+@[^ ]+/)) { - pkg = substr($0, RSTART + 3, RLENGTH - 3) - if (pkg in ignored) { + if (match($0, /on [^ ]+@/)) { + package_name = substr($0, RSTART + 3, RLENGTH - 4) + if (package_name in ignored) { skip = 1 - print "::notice::Ignoring known false-positive FOSSA vulnerability on " pkg > "/dev/stderr" + print "::notice::Ignoring known false-positive FOSSA vulnerability on " package_name > "/dev/stderr" } } } @@ -698,14 +701,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: REUSE Compliance Check - uses: fsfe/reuse-action@v1.1 + uses: fsfe/reuse-action@v6.0.0 pre-commit: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} @@ -728,19 +731,19 @@ jobs: steps: - name: Checkout if: github.event_name != 'pull_request' - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: submodules: false fetch-depth: "0" - name: Checkout for PR if: github.event_name == 'pull_request' - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: submodules: false fetch-depth: "0" ref: ${{ github.head_ref }} - name: Secret Scanning Trufflehog - uses: trufflesecurity/trufflehog@v3.88.5 + uses: trufflesecurity/trufflehog@v3.95.9 with: extra_args: -x .github/workflows/exclude-patterns.txt --json --only-verified version: 3.77.0 @@ -752,8 +755,8 @@ jobs: with: block_mode: "policy" - run-unit-tests: - name: run-unit-tests + run-unit-tests-py39: + name: run-unit-tests (3.9) runs-on: ubuntu-22.04 needs: - setup-workflow @@ -765,6 +768,8 @@ jobs: packages: read statuses: read checks: write + env: + UNIT_TESTS_PYTHON_VERSION: "3.9" steps: - uses: actions/create-github-app-token@v3 id: app-token @@ -772,15 +777,15 @@ jobs: client-id: ${{ secrets.GH_APP_CLIENT_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: ${{ env.UNIT_TESTS_PYTHON_VERSION }} - name: Setup addon run: | if [ -f "poetry.lock" ] then - python${{ env.PYTHON_VERSION }} -m pip install poetry==${{ env.POETRY_VERSION }} poetry-plugin-export==${{ env.POETRY_EXPORT_PLUGIN_VERSION }} + python${{ env.UNIT_TESTS_PYTHON_VERSION }} -m pip install poetry==${{ env.POETRY_VERSION }} poetry-plugin-export==${{ env.POETRY_EXPORT_PLUGIN_VERSION }} mkdir -p package/lib || true poetry check --lock poetry export --without-hashes --with dev -o package/lib/requirements.txt @@ -790,8 +795,8 @@ jobs: fi git config --global --add url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com".insteadOf https://github.com git config --global --add url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com".insteadOf ssh://git@github.com - python${{ env.PYTHON_VERSION }} -m venv ~/.dev_venv - ~/.dev_venv/bin/python${{ env.PYTHON_VERSION }} -m pip install -r package/lib/requirements.txt + python${{ env.UNIT_TESTS_PYTHON_VERSION }} -m venv ~/.dev_venv + ~/.dev_venv/bin/python${{ env.UNIT_TESTS_PYTHON_VERSION }} -m pip install -r package/lib/requirements.txt - name: Create directories run: | mkdir -p /opt/splunk/var/log/splunk @@ -799,7 +804,7 @@ jobs: - name: Copy pytest ini run: cp tests/unit/pytest-ci.ini pytest.ini - name: Run Pytest with coverage - run: ~/.dev_venv/bin/python${{ env.PYTHON_VERSION }} -m pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml tests/unit + run: ~/.dev_venv/bin/python${{ env.UNIT_TESTS_PYTHON_VERSION }} -m pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml tests/unit - name: Job summary continue-on-error: true run: | @@ -818,10 +823,84 @@ jobs: echo "no XML File found, exiting" exit 1 fi - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: success() || failure() with: - name: test-results-unit-python_${{ env.PYTHON_VERSION }} + name: test-results-unit-python_${{ env.UNIT_TESTS_PYTHON_VERSION }} + path: test-results/* + + run-unit-tests-py313: + name: run-unit-tests (3.13) + runs-on: ubuntu-22.04 + needs: + - setup-workflow + if: ${{ needs.setup-workflow.outputs.execute-unit == 'true' }} + permissions: + actions: read + deployments: read + contents: read + packages: read + statuses: read + checks: write + env: + UNIT_TESTS_PYTHON_VERSION: "3.13" + steps: + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.GH_APP_CLIENT_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.UNIT_TESTS_PYTHON_VERSION }} + - name: Setup addon + run: | + if [ -f "poetry.lock" ] + then + python${{ env.UNIT_TESTS_PYTHON_VERSION }} -m pip install poetry==${{ env.POETRY_VERSION }} poetry-plugin-export==${{ env.POETRY_EXPORT_PLUGIN_VERSION }} + mkdir -p package/lib || true + poetry check --lock + poetry export --without-hashes --with dev -o package/lib/requirements.txt + else + echo "No poetry.lock found, make sure your dependencies are managed through poetry, exiting" + exit 1 + fi + git config --global --add url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com".insteadOf https://github.com + git config --global --add url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com".insteadOf ssh://git@github.com + python${{ env.UNIT_TESTS_PYTHON_VERSION }} -m venv ~/.dev_venv + ~/.dev_venv/bin/python${{ env.UNIT_TESTS_PYTHON_VERSION }} -m pip install -r package/lib/requirements.txt + - name: Create directories + run: | + mkdir -p /opt/splunk/var/log/splunk + chmod -R 777 /opt/splunk/var/log/splunk + - name: Copy pytest ini + run: cp tests/unit/pytest-ci.ini pytest.ini + - name: Run Pytest with coverage + run: ~/.dev_venv/bin/python${{ env.UNIT_TESTS_PYTHON_VERSION }} -m pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml tests/unit + - name: Job summary + continue-on-error: true + run: | + sudo apt-get update + sudo apt-get install -y libxml2-utils + junit_xml_file=$(find "test-results" -name "*.xml" -type f 2>/dev/null | head -n 1) + + if [ -n "$junit_xml_file" ]; then + total_tests=$(xmllint --xpath "count(//testcase)" "$junit_xml_file") + failures=$(xmllint --xpath "count(//testcase[failure])" "$junit_xml_file") + errors=$(xmllint --xpath "count(//testcase[error and not(failure)])" "$junit_xml_file") + skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file") + passed=$((total_tests - failures - errors - skipped)) + echo -e "| Total Tests | Passed Tests | Failed Tests | Errored Tests | Skipped Tests |\n| ----------- | ------------ | ------------ | ------------- | ------------- |\n| $total_tests | $passed | $failures | $errors | $skipped |" >> "$GITHUB_STEP_SUMMARY" + else + echo "no XML File found, exiting" + exit 1 + fi + - uses: actions/upload-artifact@v7 + if: success() || failure() + with: + name: test-results-unit-python_${{ env.UNIT_TESTS_PYTHON_VERSION }} path: test-results/* build: @@ -835,9 +914,10 @@ jobs: - pre-commit - review_secrets - semgrep - - run-unit-tests + - run-unit-tests-py39 + - run-unit-tests-py313 - fossa-scan - if: ${{ !cancelled() && (needs.run-unit-tests.result == 'success' || needs.run-unit-tests.result == 'skipped') && (needs.validate-custom-version.result == 'success' || needs.validate-custom-version.result == 'skipped') && (needs.check-docs-changes.outputs.docs-only == 'false') }} + if: ${{ !cancelled() && (needs.run-unit-tests-py39.result == 'success' || needs.run-unit-tests-py39.result == 'skipped') && (needs.run-unit-tests-py313.result == 'success' || needs.run-unit-tests-py313.result == 'skipped') && (needs.validate-custom-version.result == 'success' || needs.validate-custom-version.result == 'skipped') && (needs.check-docs-changes.outputs.docs-only == 'false') }} outputs: buildname: ${{ steps.buildupload.outputs.name }} permissions: @@ -850,7 +930,7 @@ jobs: client-id: ${{ secrets.GH_APP_CLIENT_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: # Very Important semantic-release won't trigger a tagged # build if this is not set false @@ -910,7 +990,7 @@ jobs: run: if [ -f dev_deps/requirements_dev.txt ]; then echo "ENABLED=true" >> "$GITHUB_OUTPUT"; fi - name: pip cache if: ${{ steps.checklibs.outputs.ENABLED == 'true' }} - uses: actions/cache@v5 + uses: actions/cache@v6.1.0 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('dev_deps/requirements_dev.txt') }} @@ -959,13 +1039,13 @@ jobs: echo "VERSION=${FINALVERSION}" >> "$GITHUB_OUTPUT" - name: Download THIRDPARTY if: github.event_name != 'pull_request' && github.event_name != 'schedule' - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: name: THIRDPARTY - name: Download THIRDPARTY (Optional for PR and schedule) if: github.event_name == 'pull_request' || github.event_name == 'schedule' continue-on-error: true - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: name: THIRDPARTY - name: Update Notices @@ -1015,13 +1095,13 @@ jobs: echo "OUTPUT=$PACKAGE" >> "$GITHUB_OUTPUT" if: always() - name: artifact-openapi - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: artifact-openapi path: ${{ github.workspace }}/${{ steps.uccgen.outputs.OUTPUT }}/appserver/static/openapi.json if: ${{ !cancelled() && needs.setup-workflow.outputs.has-ucc_modinput-tests == 'true' }} - name: artifact-splunk-base - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: package-splunkbase path: ${{ steps.slim.outputs.OUTPUT }} @@ -1037,12 +1117,129 @@ jobs: basename "${{ steps.slim.outputs.OUTPUT }}" aws s3 cp "${{ steps.slim.outputs.OUTPUT }}" "s3://${{ needs.setup-workflow.outputs.s3_bucket_k8s }}/ta-apps/" - name: artifact-splunk-parts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: package-deployment path: build/package/deployment** if: ${{ !cancelled() }} + build-py313: + name: build (3.13) + runs-on: ubuntu-22.04 + needs: + - check-docs-changes + - validate-custom-version + - setup-workflow + - meta + - compliance-copyrights + - pre-commit + - review_secrets + - semgrep + - run-unit-tests-py313 + - fossa-scan + if: ${{ !cancelled() && (needs.run-unit-tests-py313.result == 'success' || needs.run-unit-tests-py313.result == 'skipped') && (needs.validate-custom-version.result == 'success' || needs.validate-custom-version.result == 'skipped') && (needs.check-docs-changes.outputs.docs-only == 'false') }} + permissions: + contents: read + packages: read + env: + BUILD_PYTHON_VERSION: "3.13" + steps: + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.GH_APP_CLIENT_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Setup python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.BUILD_PYTHON_VERSION }} + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + - name: create requirements file for pip + run: | + git config --global --add url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com".insteadOf https://github.com + git config --global --add url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com".insteadOf ssh://git@github.com + if [ -f "poetry.lock" ] + then + echo " poetry.lock found " + python${{ env.BUILD_PYTHON_VERSION }} -m pip install poetry==${{ env.POETRY_VERSION }} poetry-plugin-export==${{ env.POETRY_EXPORT_PLUGIN_VERSION }} + poetry export --without-hashes -o requirements.txt + if [ "$(grep -cve '^\s*$' requirements.txt)" -ne 0 ] + then + echo "Prod dependencies were found, creating package/lib folder" + mkdir -p package/lib || true + mv requirements.txt package/lib + else + echo "No prod dependencies were found" + rm requirements.txt + fi + fi + - name: Setup UCC + run: | + if [ -f "dev_deps/requirements_ucc.txt" ]; then + echo "Found requirements_ucc.txt. Installing UCC dependencies in an isolated environment" + python${{ env.BUILD_PYTHON_VERSION }} -m venv ~/.ucc_venv + ~/.ucc_venv/bin/python${{ env.BUILD_PYTHON_VERSION }} -m pip install -r dev_deps/requirements_ucc.txt + export UCC_GEN="$HOME/.ucc_venv/bin/ucc-gen" + + if [ ! -f "$UCC_GEN" ]; then + echo "ucc-gen not found after installing requirements from dev_deps/requirements_ucc.txt" + exit 1 + fi + + echo "UCC_GEN=$UCC_GEN" >> "$GITHUB_ENV" + else + echo "No UCC requirements file found, skipping UCC setup" + echo "Create a dev_deps/requirements_ucc.txt file and place UCC requirement there, to avoid dependency conflicts" + exit 1 + fi + - name: Download THIRDPARTY + if: github.event_name != 'pull_request' && github.event_name != 'schedule' + uses: actions/download-artifact@v8 + with: + name: THIRDPARTY + - name: Download THIRDPARTY (Optional for PR and schedule) + if: github.event_name == 'pull_request' || github.event_name == 'schedule' + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: THIRDPARTY + - name: Update Notices + run: | + cp -f THIRDPARTY package/THIRDPARTY || echo "THIRDPARTY file not found (allowed for PR and schedule)" + - name: Build Package + id: uccgen + run: | + "${UCC_GEN:-ucc-gen}" build --ta-version 0.0.${{ github.run_id }} + if [ -f package.json ] + then + yarn + yarn version --new-version 0.0.${{ github.run_id }} --no-git-tag-version + yarn run build + fi + echo "OUTPUT=output/$(ls output/)" >> "$GITHUB_OUTPUT" + - name: Slim Package + id: slim + run: | + pip install splunk-packaging-toolkit==1.2.4 + INPUT_SOURCE=${{ steps.uccgen.outputs.OUTPUT }} + SOURCE_REGEX='^.*/$' + if [[ $INPUT_SOURCE =~ $SOURCE_REGEX ]];then + echo Removing trailing / from INPUT_SOURCE slim is picky + INPUT_SOURCE=$(echo $INPUT_SOURCE | sed 's/\(.*\)\//\1/') + fi + slim generate-manifest "${INPUT_SOURCE}" --update >/tmp/app.manifest || true + cp /tmp/app.manifest "${INPUT_SOURCE}"/app.manifest + mkdir -p build/package/splunkbase + slim package -o build/package/splunkbase "${INPUT_SOURCE}" + slim validate build/package/splunkbase/* + if: always() appinspect: name: quality-appinspect-${{ matrix.tags }} @@ -1061,26 +1258,26 @@ jobs: - "self-service" - "manual" steps: - - uses: actions/checkout@v5 - - uses: actions/download-artifact@v7 + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 with: name: package-splunkbase path: build/package/ - name: Scan - uses: splunk/appinspect-cli-action@v2.11 + uses: splunk/appinspect-cli-action@v2.13.0 with: app_path: build/package/ included_tags: ${{ matrix.tags }} result_file: appinspect_result_${{ matrix.tags }}.json - name: upload-appinspect-report if: ${{ !cancelled() }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: appinspect_${{ matrix.tags }}_checks.json path: appinspect_result_${{ matrix.tags }}.json - name: upload-markdown if: matrix.tags == 'manual' - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: check_markdown path: | @@ -1100,8 +1297,8 @@ jobs: tags: - "cloud" steps: - - uses: actions/checkout@v5 - - uses: actions/download-artifact@v7 + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 with: name: package-splunkbase path: build/package @@ -1112,7 +1309,7 @@ jobs: password: ${{ secrets.SPL_COM_PASSWORD }} app_path: build/package/ included_tags: ${{ matrix.tags }} - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: always() with: name: appinspect-api-html-report-${{ matrix.tags }} @@ -1132,7 +1329,7 @@ jobs: client-id: ${{ secrets.GH_APP_CLIENT_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 with: @@ -1163,7 +1360,7 @@ jobs: -v "$(pwd)":/addon \ 956110764581.dkr.ecr.us-west-2.amazonaws.com/ta-automation/gs-scorecard:"${{ env.GS_IMAGE_VERSION }}" - name: Upload GS Scorecard report - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: gs-scorecard-report path: ./gs_scorecard.json @@ -1213,7 +1410,7 @@ jobs: client-id: ${{ secrets.GH_APP_CLIENT_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive token: ${{ steps.app-token.outputs.token }} @@ -1251,7 +1448,7 @@ jobs: echo "spl-host-suffix=wfe.splgdi.com" echo "k8s-manifests-branch=${{ inputs.k8s-manifests-branch }}" } >> "$GITHUB_OUTPUT" - - uses: actions/download-artifact@v7 + - uses: actions/download-artifact@v8 if: ${{ needs.setup-workflow.outputs.has-ucc_modinput-tests == 'true' }} id: download-openapi with: @@ -1341,7 +1538,7 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive - name: capture start time @@ -1374,7 +1571,7 @@ jobs: timeout-minutes: 20 env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v5.2 + uses: splunk/wfe-test-runner-action@v5.3 with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} @@ -1389,6 +1586,7 @@ jobs: sc4s-docker-registry: ${{ matrix.sc4s.docker_registry }} k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} python-version: ${{ env.PYTHON_VERSION }} + server-conf-python-version: '' - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation id: update-argo-token if: ${{ !cancelled() }} @@ -1493,13 +1691,13 @@ jobs: echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/workflows/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() && steps.check-workflow-status.outputs.workflow-status != 'Succeeded' }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() && steps.check-workflow-status.outputs.workflow-status != 'Succeeded' }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests logs @@ -1521,7 +1719,7 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive - name: configure git # This step configures git to omit "dubious git ownership error" in later test-reporter stage @@ -1543,7 +1741,8 @@ jobs: name: spl2 test report path: "test-results/*.xml" reporter: java-junit - - uses: actions/upload-artifact@v4 + use-actions-summary: false + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: spl2 test results @@ -1581,7 +1780,7 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive - name: configure git # This step configures git to omit "dubious git ownership error" in later test-reporter stage @@ -1622,7 +1821,7 @@ jobs: continue-on-error: true env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v5.2 + uses: splunk/wfe-test-runner-action@v5.3 with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} @@ -1637,6 +1836,7 @@ jobs: sc4s-docker-registry: ${{ matrix.sc4s.docker_registry }} k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} python-version: ${{ env.PYTHON_VERSION }} + server-conf-python-version: '' - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation id: update-argo-token if: ${{ !cancelled() }} @@ -1734,20 +1934,20 @@ jobs: echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/workflows/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests logs path: | ${{ needs.setup.outputs.directory-path }}/argo-logs - name: Upload cim-compliance-report for ${{ matrix.splunk.version }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 if: ${{ matrix.splunk.islatest == true }} with: name: cim-compliance-report @@ -1761,6 +1961,7 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + use-actions-summary: false - name: Parse JUnit XML if: ${{ !cancelled() }} run: | @@ -1780,7 +1981,7 @@ jobs: exit 1 fi - name: Upload-artifact-for-github-summary - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: summary-ko-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} @@ -1790,7 +1991,7 @@ jobs: run: | echo "pulling diag" aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/diag-${{ steps.create-job-name.outputs.job-name }}/diag-${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} tests diag @@ -1803,7 +2004,7 @@ jobs: if: ${{ !cancelled() && needs.run-knowledge-tests.result != 'skipped' }} steps: - name: Download all summaries - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: pattern: summary-ko* - name: Combine summaries into a table @@ -1853,7 +2054,7 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive - name: configure git # This step configures git to omit "dubious git ownership error" in later test-reporter stage @@ -1906,7 +2107,7 @@ jobs: continue-on-error: true env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v5.2 + uses: splunk/wfe-test-runner-action@v5.3 with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} @@ -1922,6 +2123,7 @@ jobs: k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} test-browser: ${{ env.TEST_BROWSER }} python-version: ${{ env.PYTHON_VERSION }} + server-conf-python-version: '' - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation id: update-argo-token if: ${{ !cancelled() }} @@ -2019,13 +2221,13 @@ jobs: echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/workflows/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.browser }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.browser }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests logs @@ -2039,6 +2241,7 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.browser }} ${{ matrix.vendor-version.image }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + use-actions-summary: false - name: Parse JUnit XML if: ${{ !cancelled() }} run: | @@ -2058,7 +2261,7 @@ jobs: exit 1 fi - name: Upload-artifact-for-github-summary - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.browser }}-${{ matrix.vendor-version.image }}-${{ matrix.marker }}-artifact @@ -2068,7 +2271,7 @@ jobs: run: | echo "pulling diag" aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/diag-${{ steps.create-job-name.outputs.job-name }}/diag-${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.browser }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests diag @@ -2081,7 +2284,7 @@ jobs: if: ${{ !cancelled() && needs.run-ui-tests.result != 'skipped' }} steps: - name: Download all summaries - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: pattern: summary-ui* - name: Combine summaries into a table @@ -2107,7 +2310,7 @@ jobs: strategy: fail-fast: false matrix: - splunk: ${{ fromJson(needs.meta.outputs.matrix_Splunk) }} + splunk: ${{ fromJson(needs.meta.outputs.matrix_splunkModinput) }} modinput-type: [ "modinput_functional" ] vendor-version: ${{ fromJson(needs.meta.outputs.matrix_supportedModinputFunctionalVendors) }} marker: ${{ fromJson(inputs.marker) }} @@ -2130,7 +2333,7 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive - name: configure git # This step configures git to omit "dubious git ownership error" in later test-reporter stage @@ -2183,7 +2386,7 @@ jobs: continue-on-error: true env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v5.2 + uses: splunk/wfe-test-runner-action@v5.3 with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} @@ -2198,6 +2401,7 @@ jobs: sc4s-version: "No" k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} python-version: ${{ env.PYTHON_VERSION }} + server-conf-python-version: ${{ matrix.splunk.serverConfPythonVersion || '' }} - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation id: update-argo-token if: ${{ !cancelled() }} @@ -2295,16 +2499,16 @@ jobs: echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/workflows/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests artifacts + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.splunk.serverConfPythonVersion || 'default' }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests logs + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.splunk.serverConfPythonVersion || 'default' }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests logs path: | ${{ needs.setup.outputs.directory-path }}/argo-logs - name: Test Report @@ -2315,6 +2519,7 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + use-actions-summary: false - name: Parse JUnit XML if: ${{ !cancelled() }} run: | @@ -2334,20 +2539,20 @@ jobs: exit 1 fi - name: Upload-artifact-for-github-summary - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: - name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.vendor-version.image }}-${{ matrix.marker }}-artifact + name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.splunk.serverConfPythonVersion || 'default' }}-${{ matrix.vendor-version.image }}-${{ matrix.marker }}-artifact path: job_summary.txt - name: pull diag from s3 bucket if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} run: | echo "pulling diag" aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/diag-${{ steps.create-job-name.outputs.job-name }}/diag-${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests diag + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.splunk.serverConfPythonVersion || 'default' }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests diag path: | ${{ needs.setup.outputs.directory-path }}/diag* @@ -2357,7 +2562,7 @@ jobs: if: ${{ !cancelled() && needs.run-modinput-tests.result != 'skipped' }} steps: - name: Download all summaries - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: pattern: summary-modinput* - name: Combine summaries into a table @@ -2383,7 +2588,7 @@ jobs: strategy: fail-fast: false matrix: - splunk: ${{ fromJson(needs.meta.outputs.matrix_Splunk) }} + splunk: ${{ fromJson(needs.meta.outputs.matrix_splunkModinput) }} vendor-version: ${{ fromJson(needs.meta.outputs.matrix_supportedModinputFunctionalVendors) }} marker: ${{ fromJson(inputs.ucc-modinput-marker) }} container: @@ -2405,7 +2610,7 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive - name: configure git # This step configures git to omit "dubious git ownership error" in later test-reporter stage @@ -2458,7 +2663,7 @@ jobs: continue-on-error: true env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v5.2 + uses: splunk/wfe-test-runner-action@v5.3 with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} @@ -2473,6 +2678,7 @@ jobs: sc4s-version: "No" k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} python-version: ${{ env.PYTHON_VERSION }} + server-conf-python-version: ${{ matrix.splunk.serverConfPythonVersion || '' }} - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation id: update-argo-token if: ${{ !cancelled() }} @@ -2570,16 +2776,16 @@ jobs: echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/workflows/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests artifacts + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.splunk.serverConfPythonVersion || 'default' }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests logs + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.splunk.serverConfPythonVersion || 'default' }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests logs path: | ${{ needs.setup.outputs.directory-path }}/argo-logs - name: Test Report @@ -2590,6 +2796,7 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + use-actions-summary: false - name: Parse JUnit XML if: ${{ !cancelled() }} run: | @@ -2609,20 +2816,20 @@ jobs: exit 1 fi - name: Upload-artifact-for-github-summary - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: - name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.vendor-version.image }}-${{ matrix.marker }}-artifact + name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.splunk.serverConfPythonVersion || 'default' }}-${{ matrix.vendor-version.image }}-${{ matrix.marker }}-artifact path: job_summary.txt - name: pull diag from s3 bucket if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} run: | echo "pulling diag" aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/diag-${{ steps.create-job-name.outputs.job-name }}/diag-${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} with: - name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests diag + name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.splunk.serverConfPythonVersion || 'default' }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} tests diag path: | ${{ needs.setup.outputs.directory-path }}/diag* @@ -2632,7 +2839,7 @@ jobs: if: ${{ !cancelled() && needs.run-ucc-modinput-tests.result != 'skipped' }} steps: - name: Download all summaries - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: pattern: summary-ucc_modinput* - name: Combine summaries into a table @@ -2680,7 +2887,7 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive - name: configure git # This step configures git to omit "dubious git ownership error" in later test-reporter stage @@ -2721,7 +2928,7 @@ jobs: continue-on-error: true env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v5.2 + uses: splunk/wfe-test-runner-action@v5.3 with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} @@ -2737,6 +2944,7 @@ jobs: k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} ta-upgrade-version: ${{ matrix.ta-version-from-upgrade }} python-version: ${{ env.PYTHON_VERSION }} + server-conf-python-version: '' - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation id: update-argo-token if: ${{ !cancelled() }} @@ -2834,13 +3042,13 @@ jobs: echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.ta-version-from-upgrade }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ matrix.ta-version-from-upgrade }} tests logs @@ -2854,6 +3062,7 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + use-actions-summary: false - name: Parse JUnit XML if: ${{ !cancelled() }} run: | @@ -2873,7 +3082,7 @@ jobs: exit 1 fi - name: Upload-artifact-for-github-summary - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.vendor-version.image }}-${{ matrix.ta-version-from-upgrade }}-artifact @@ -2883,7 +3092,7 @@ jobs: run: | echo "pulling diag" aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/diag-${{ steps.create-job-name.outputs.job-name }}/diag-${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} tests diag @@ -2896,7 +3105,7 @@ jobs: if: ${{ !cancelled() && needs.run-upgrade-tests.result != 'skipped' }} steps: - name: Download all summaries - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: pattern: summary-upgrade* - name: Combine summaries into a table @@ -2942,7 +3151,7 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive - name: configure git # This step configures git to omit "dubious git ownership error" in later test-reporter stage @@ -2996,7 +3205,7 @@ jobs: continue-on-error: true env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v5.2 + uses: splunk/wfe-test-runner-action@v5.3 with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} @@ -3012,6 +3221,7 @@ jobs: os-version: ${{ steps.os-name-version.outputs.os-version }} k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} python-version: ${{ env.PYTHON_VERSION }} + server-conf-python-version: '' - name: calculate timeout id: calculate-timeout run: | @@ -3103,13 +3313,13 @@ jobs: echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/workflows/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ steps.os-name-version.outputs.os-name }} ${{ steps.os-name-version.outputs.os-version }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ steps.os-name-version.outputs.os-name }} ${{ steps.os-name-version.outputs.os-version }} tests logs @@ -3123,6 +3333,7 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ steps.os-name-version.outputs.os-name }} ${{ steps.os-name-version.outputs.os-version }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + use-actions-summary: false - name: Parse JUnit XML if: ${{ !cancelled() }} run: | @@ -3142,7 +3353,7 @@ jobs: exit 1 fi - name: Upload-artifact-for-github-summary - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ steps.os-name-version.outputs.os-name }}-${{ steps.os-name-version.outputs.os-version }} @@ -3152,7 +3363,7 @@ jobs: run: | echo "pulling diag" aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/diag-${{ steps.create-job-name.outputs.job-name }}/diag-${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ steps.os-name-version.outputs.os-name }} ${{ steps.os-name-version.outputs.os-version }} tests diag @@ -3165,7 +3376,7 @@ jobs: if: ${{ !cancelled() && needs.run-scripted-input-tests-full-matrix.result != 'skipped' }} steps: - name: Download all summaries - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: pattern: summary-scripted* - name: Combine summaries into a table @@ -3212,7 +3423,7 @@ jobs: statuses: read checks: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: submodules: recursive - name: configure git # This step configures git to omit "dubious git ownership error" in later test-reporter stage @@ -3253,7 +3464,7 @@ jobs: continue-on-error: true env: ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }} - uses: splunk/wfe-test-runner-action@v5.2 + uses: splunk/wfe-test-runner-action@v5.3 with: splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} test-type: ${{ env.TEST_TYPE }} @@ -3268,6 +3479,7 @@ jobs: sc4s-version: "No" k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }} python-version: ${{ env.PYTHON_VERSION }} + server-conf-python-version: '' - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation id: update-argo-token if: ${{ !cancelled() }} @@ -3365,13 +3577,13 @@ jobs: echo "pulling logs" mkdir -p ${{ needs.setup.outputs.directory-path }}/argo-logs aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/${WORKFLOW_NAME}/ ${{ needs.setup.outputs.directory-path }}/argo-logs/ --recursive - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ env.SPL2_TEST_TYPE }} tests artifacts path: | ${{ needs.setup.outputs.directory-path }}/test-results - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ env.SPL2_TEST_TYPE }} tests logs @@ -3385,6 +3597,7 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ env.SPL2_TEST_TYPE }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + use-actions-summary: false - name: Parse JUnit XML if: ${{ !cancelled() }} run: | @@ -3404,7 +3617,7 @@ jobs: exit 1 fi - name: Upload-artifact-for-github-summary - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: summary-${{ env.TEST_TYPE }}-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ env.SPL2_TEST_TYPE }}-artifact @@ -3416,7 +3629,7 @@ jobs: if: ${{ !cancelled() && needs.run-spl2-integration-tests.result != 'skipped' }} steps: - name: Download all summaries - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: pattern: summary-spl2_integration_tests* - name: Combine summaries into a table @@ -3448,7 +3661,9 @@ jobs: - review_secrets - semgrep - build - - run-unit-tests + - build-py313 + - run-unit-tests-py39 + - run-unit-tests-py313 - appinspect - setup - run-knowledge-tests @@ -3490,7 +3705,7 @@ jobs: statuses: write steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: submodules: false persist-credentials: false @@ -3516,14 +3731,14 @@ jobs: make_latest: false - name: Download package-deployment if: ${{ steps.semantic.outputs.new_release_published == 'true' || steps.custom.outputs.upload_url != '' }} - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 id: download-package-deployment with: name: package-deployment path: download/artifacts/ - name: Download package-splunkbase if: ${{ steps.semantic.outputs.new_release_published == 'true' || steps.custom.outputs.upload_url != '' }} - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 id: download-package-splunkbase with: name: package-splunkbase @@ -3532,7 +3747,7 @@ jobs: id: download-cim-compliance-report if: ${{ steps.semantic.outputs.new_release_published == 'true' || steps.custom.outputs.upload_url != '' }} continue-on-error: true - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: name: cim-compliance-report path: download/artifacts/deployment diff --git a/.github/workflows/reusable-publish-to-splunkbase.yml b/.github/workflows/reusable-publish-to-splunkbase.yml index 2bfc23a6..896c0377 100644 --- a/.github/workflows/reusable-publish-to-splunkbase.yml +++ b/.github/workflows/reusable-publish-to-splunkbase.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - id: matrix - uses: splunk/addonfactory-test-matrix-action@v3.1 + uses: splunk/addonfactory-test-matrix-action@v3.3 with: features: PYTHON39 publish: @@ -35,7 +35,7 @@ jobs: needs: - inputs-validator steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/.github/workflows/reusable-validate-deploy-docs.yml b/.github/workflows/reusable-validate-deploy-docs.yml index b772bde2..02c52721 100644 --- a/.github/workflows/reusable-validate-deploy-docs.yml +++ b/.github/workflows/reusable-validate-deploy-docs.yml @@ -9,7 +9,7 @@ jobs: outputs: status: ${{ steps.validate.outputs.status }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: 3.12 @@ -40,7 +40,7 @@ jobs: pages: write if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: 3.12 diff --git a/README.md b/README.md index cd909206..5d2f103d 100644 --- a/README.md +++ b/README.md @@ -184,12 +184,13 @@ gitGraph * `custom-version` - version used for release on manual workflow trigger (format: `x.x.x`) * `execute-tests-on-push-to-release` - enable tests on push to `release/*` branch - default `false` * `k8s-environment` - k8s environment for testing, choices: `production` (default) or `staging` -* `k8s-manifests-branch` - k8s-manifests branch for testing, default `v4.0` +* `k8s-manifests-branch` - k8s-manifests branch for testing, default `v4.3` * `scripted-inputs-os-list` - list of OSes used for scripted inputs tests (default includes ubuntu 16.04–24.04 and redhat 8.4–9.5) * `upgrade-tests-ta-versions` - list of TA versions (format `X.X.X`) used as starting points for upgrade tests; e.g. `['7.6.0', '7.7.0']` * `wfe-run-on-splunk-latest` - when `true` forces WFE tests to run only on the latest Splunk version; when `false` runs on all supported Splunk versions required for release; default `false` * `python-version` - Python version used for testing, default `3.9` -* `gs-image-version` - version of the GS Scorecard Docker image, default `1.1` +* `spl2-generate` - when `true` enables SPL2 generation, default `false` +* `gs-image-version` - version of the GS Scorecard Docker image, default `1.2` * `gs-version` - version of the GS Scorecard tool, default `0.3` ## General troubleshooting @@ -309,8 +310,8 @@ gitGraph **Description:** - Determines which Splunk and SC4S versions to run tests with. -- Outputs matrices for supported and latest Splunk versions, SC4S versions, and vendor matrices for modinput/UI tests. -- On schedule events, always uses latest Splunk only. On PRs to `main` or push to `main`/`develop`, uses the full supported matrix (unless overridden by `wfe-run-on-splunk-latest` input). +- Outputs matrices for supported and latest Splunk versions, SC4S versions, and vendor matrices for modinput/UI tests. The modinput matrix may expand a Splunk version into multiple entries when `server_conf_python_versions` variants (e.g. `python3`, `force_python3`) are configured, each run with the matching `python.version` set in `server.conf`. +- On schedule events, always uses latest Splunk only. On PRs to `main` or `release/*`, or push to `main`/`develop`/`release/*` (the latter only when `execute-tests-on-push-to-release` is `true`), uses the full supported matrix (unless overridden by `wfe-run-on-splunk-latest` input). ## [Job] fossa-scan @@ -362,6 +363,8 @@ fossa-test-output - It publishes vulnerability-only results in the CI job summary and job log so CI can distinguish security findings from license compliance findings. +- Known false positives are read from the calling repository's `package/lib/exclude.txt`, with one package name per line (for example, `urllib3`). Repositories without this file use no exceptions. + **Pass/fail behaviour:** - This stage fails if any active FOSSA vulnerability issues are found at any severity level. CI continues regardless (`continue-on-error: true`) — the failure is surfaced via the job summary and log. @@ -667,7 +670,7 @@ appinspect-api-html-report-self-service - `GH_APP_PRIVATE_KEY` (secret) and `GH_APP_CLIENT_ID` (variable) for GitHub App authentication, and `SA_GH_USER_NAME` for GitHub access - `SPL_COM_USER` and `SPL_COM_PASSWORD` for AppInspect integration -- Check that the Docker image version specified via the `gs-image-version` workflow input (`GS_IMAGE_VERSION` env var, default `1.1`) exists in the ECR registry. The GS Scorecard tool version is controlled separately via `gs-version` input (`GS_VERSION` env var, default `0.3`). +- Check that the Docker image version specified via the `gs-image-version` workflow input (`GS_IMAGE_VERSION` env var, default `1.2`) exists in the ECR registry. The GS Scorecard tool version is controlled separately via `gs-version` input (`GS_VERSION` env var, default `0.3`). - Review the job logs for specific error messages from the GS Scorecard tool. @@ -692,7 +695,7 @@ gs-scorecard-report (gs_scorecard.html) **Description:** -- This stage does the setup for executing unit tests and reports the results +- Unit tests run in two parallel jobs, `run-unit-tests-py39` and `run-unit-tests-py313`, executing the same suite against Python 3.9 and 3.13 respectively. **Action used:** NA