diff --git a/.github/workflows/real-time-benchmark.yml b/.github/workflows/real-time-benchmark.yml index 5d4c455acc1..5156afc37b5 100644 --- a/.github/workflows/real-time-benchmark.yml +++ b/.github/workflows/real-time-benchmark.yml @@ -2,14 +2,81 @@ name: Real-time Benchmark on: schedule: - cron: "30 0 * * *" + workflow_dispatch: + inputs: + pull_request: + description: 'PR number that is going to be benchmarked (e.g. "1234")' + required: true + type: number + jit: + description: 'Whether JIT is benchmarked' + required: false + type: choice + options: + - "0" + - "1" + opcache: + description: 'Whether opcache is enabled for the benchmarked commit' + required: true + default: "1" + type: choice + options: + - "0" + - "1" + - "2" + baseline_opcache: + description: 'Whether opcache is enabled for the baseline commit' + required: true + default: "1" + type: choice + options: + - "0" + - "1" + - "2" permissions: contents: read + pull-requests: write jobs: REAL_TIME_BENCHMARK: name: REAL_TIME_BENCHMARK - if: github.repository == 'php/php-src' + if: github.repository == 'php/php-src' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-22.04 + env: + REPOSITORY: ${{ github.repository }} + BRANCH: "master" + COMMIT: ${{ github.sha }} + BASELINE_COMMIT: "d5f6e56610c729710073350af318c4ea1b292cfe" + ID: "master" + OPCACHE: ${{ inputs.opcache || '1' }} + BASELINE_OPCACHE: ${{ inputs.baseline_opcache || '2' }} + JIT: ${{ inputs.jit || '1' }} + YEAR: "" steps: + - name: Setup benchmark environment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + YEAR="$(date '+%Y')" + echo "YEAR=$YEAR" >> $GITHUB_ENV + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + PR_INFO=$(gh pr view ${{ inputs.pull_request }} --json headRepositoryOwner,headRepository,headRefName,headRefOid,baseRefOid --repo ${{ github.repository }} | jq -r '.headRepositoryOwner.login, .headRepository.name, .headRefName, .headRefOid, .baseRefOid') + + REPOSITORY="$(echo "$PR_INFO" | sed -n '1p')/$(echo "$PR_INFO" | sed -n '2p')" + echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV + + BRANCH=$(echo "$PR_INFO" | sed -n '3p') + echo "BRANCH=$BRANCH" >> $GITHUB_ENV + + COMMIT=$(echo "$PR_INFO" | sed -n '4p') + echo "COMMIT=$COMMIT" >> $GITHUB_ENV + + BASELINE_COMMIT=$(echo "$PR_INFO" | sed -n '5p') + echo "BASELINE_COMMIT=$BASELINE_COMMIT" >> $GITHUB_ENV + + echo "ID=benchmarked" >> $GITHUB_ENV + fi + - name: Install dependencies run: | set -ex @@ -29,73 +96,142 @@ jobs: ref: 'main' fetch-depth: 1 path: 'php-version-benchmarks' - - name: Checkout php-src + - name: Checkout php-src (benchmarked version) uses: actions/checkout@v4 with: - repository: 'php/php-src' - ref: '${{ github.sha }}' + repository: '${{ env.REPOSITORY }}' + ref: '${{ env.COMMIT }}' fetch-depth: 100 - path: 'php-version-benchmarks/tmp/php_master' + path: 'php-version-benchmarks/tmp/php_${{ env.ID }}' + - name: Checkout php-src (baseline version) + uses: actions/checkout@v4 + with: + repository: '${{ env.REPOSITORY }}' + ref: '${{ env.BASELINE_COMMIT }}' + fetch-depth: 100 + path: 'php-version-benchmarks/tmp/php_baseline' - name: Setup benchmark results run: | git config --global user.name "Benchmark" git config --global user.email "benchmark@php.net" - + rm -rf ./php-version-benchmarks/docs/results - name: Checkout benchmark data + if: github.event_name != 'workflow_dispatch' uses: actions/checkout@v4 with: repository: php/real-time-benchmark-data ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }} path: 'php-version-benchmarks/docs/results' - - name: Set benchmark config + - name: Setup infra config run: | set -e - # Set infrastructure config cp ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini.dist ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini ESCAPED_DOCKER_REGISTRY=$(printf '%s\n' "${{ secrets.PHP_VERSION_BENCHMARK_DOCKER_REGISTRY }}" | sed -e 's/[\/&]/\\&/g') sed -i "s/INFRA_DOCKER_REGISTRY=public.ecr.aws\/abcdefgh/INFRA_DOCKER_REGISTRY=$ESCAPED_DOCKER_REGISTRY/g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini cp ./php-version-benchmarks/build/infrastructure/config/aws.tfvars.dist ./php-version-benchmarks/build/infrastructure/config/aws.tfvars sed -i 's/access_key = ""/access_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_ACCESS_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars sed -i 's/secret_key = ""/secret_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_SECRET_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars + sed -i 's/github_token = ""/github_token = "${{ secrets.GITHUB_TOKEN }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars + - name: Setup PHP config - baseline PHP version + run: | + set -e - YEAR="$(date '+%Y')" - DATABASE="./php-version-benchmarks/docs/results/$YEAR/database.tsv" + BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c1-4)" + + cat << EOF > ./php-version-benchmarks/config/php/baseline.ini + PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA" + PHP_ID=php_baseline + + PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git + PHP_BRANCH=${{ env.BRANCH }} + PHP_COMMIT=${{ env.BASELINE_COMMIT }} + + PHP_OPCACHE=${{ env.BASELINE_OPCACHE }} + PHP_JIT=0 + EOF + - name: Setup PHP config - baseline PHP version with JIT + if: github.event_name == 'workflow_dispatch' && inputs.jit == '1' + run: | + set -e + + BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c1-4)" + + cat << EOF > ./php-version-benchmarks/config/php/baseline_jit.ini + PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA (JIT)" + PHP_ID=php_baseline_jit + + PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git + PHP_BRANCH=${{ env.BRANCH }} + PHP_COMMIT=${{ env.BASELINE_COMMIT }} + + PHP_OPCACHE=${{ env.BASELINE_OPCACHE }} + PHP_JIT=${{ env.JIT }} + EOF + + git clone ./php-version-benchmarks/tmp/php_baseline/ ./php-version-benchmarks/tmp/php_baseline_jit + - name: Setup PHP config - previous PHP version + if: github.event_name != 'workflow_dispatch' + run: | + set -e + + DATABASE="./php-version-benchmarks/docs/results/${{ env.YEAR }}/database.tsv" if [ -f "$DATABASE" ]; then LAST_RESULT_SHA="$(tail -n 2 "$DATABASE" | head -n 1 | cut -f 6)" else YESTERDAY="$(date -d "-2 day 23:59:59" '+%Y-%m-%d %H:%M:%S')" - LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_master/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')" + LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_${{ env.ID }}/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')" fi - BASELINE_SHA="d5f6e56610c729710073350af318c4ea1b292cfe" - BASELINE_SHORT_SHA="$(echo "$BASELINE_SHA" | cut -c1-4)" + cat << EOF > ./php-version-benchmarks/config/php/previous.ini + PHP_NAME="PHP - previous ${{ env.BRANCH }}" + PHP_ID=php_previous - # Set config for the baseline PHP version - cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_baseline.ini - sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - baseline@'"$BASELINE_SHORT_SHA"'"/g' ./php-version-benchmarks/config/php/master_baseline.ini - sed -i "s/PHP_ID=php_master/PHP_ID=php_master_baseline/g" ./php-version-benchmarks/config/php/master_baseline.ini - sed -i "s/PHP_COMMIT=/PHP_COMMIT=$BASELINE_SHA/g" ./php-version-benchmarks/config/php/master_baseline.ini - sed -i "s/PHP_OPCACHE=1/PHP_OPCACHE=2/g" ./php-version-benchmarks/config/php/master_baseline.ini + PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git + PHP_BRANCH=${{ env.BRANCH }} + PHP_COMMIT=$LAST_RESULT_SHA - # Set config for the previous PHP version - cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_last.ini - sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - previous master"/g' ./php-version-benchmarks/config/php/master_last.ini - sed -i "s/PHP_ID=php_master/PHP_ID=php_master_previous/g" ./php-version-benchmarks/config/php/master_last.ini - sed -i "s/PHP_COMMIT=/PHP_COMMIT=$LAST_RESULT_SHA/g" ./php-version-benchmarks/config/php/master_last.ini - sed -i "s/PHP_OPCACHE=1/PHP_OPCACHE=2/g" ./php-version-benchmarks/config/php/master_last.ini + PHP_OPCACHE=1 + PHP_JIT=0 + EOF + - name: Setup PHP config - benchmarked PHP version + run: | + set -e - # Set config for the current PHP version - cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_now.ini - sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master_now.ini + cat << EOF > ./php-version-benchmarks/config/php/this.ini + PHP_NAME="PHP - ${{ env.BRANCH }}" + PHP_ID=php_${{ env.ID }} - # Set config for current PHP version with JIT - git clone ./php-version-benchmarks/tmp/php_master/ ./php-version-benchmarks/tmp/php_master_jit - cp ./php-version-benchmarks/config/php/master_jit.ini.dist ./php-version-benchmarks/config/php/master_now_jit.ini - sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master_now_jit.ini + PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git + PHP_BRANCH=${{ env.BRANCH }} + PHP_COMMIT=${{ env.COMMIT }} + + PHP_OPCACHE=${{ env.OPCACHE }} + PHP_JIT=0 + EOF + - name: Setup PHP config - benchmarked PHP version with JIT + if: env.JIT == '1' + run: | + set -e + + cat << EOF > ./php-version-benchmarks/config/php/this_jit.ini + PHP_NAME="PHP - ${{ env.BRANCH }} (JIT)" + PHP_ID=php_${{ env.ID }}_jit + + PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git + PHP_BRANCH=${{ env.BRANCH }} + PHP_COMMIT=${{ env.COMMIT }} + + PHP_OPCACHE=${{ env.OPCACHE }} + PHP_JIT=${{ env.JIT }} + EOF + + git clone ./php-version-benchmarks/tmp/php_${{ env.ID }}/ ./php-version-benchmarks/tmp/php_${{ env.ID }}_jit + - name: Setup test config + run: | + set -e - # Set test configs cp ./php-version-benchmarks/config/test/1_laravel.ini.dist ./php-version-benchmarks/config/test/1_laravel.ini cp ./php-version-benchmarks/config/test/2_symfony_main.ini.dist ./php-version-benchmarks/config/test/2_symfony_main.ini cp ./php-version-benchmarks/config/test/4_wordpress.ini.dist ./php-version-benchmarks/config/test/4_wordpress.ini @@ -104,6 +240,7 @@ jobs: - name: Run benchmark run: ./php-version-benchmarks/benchmark.sh run aws - name: Store results + if: github.repository == 'php/php-src' && github.event_name != 'workflow_dispatch' run: | set -ex @@ -119,6 +256,21 @@ jobs: fi git commit -m "Add result for ${{ github.repository }}@${{ github.sha }}" git push + - name: Upload artifact + if: github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v4 + with: + name: results + path: ./php-version-benchmarks/docs/results/${{ env.YEAR }} + retention-days: 30 + - name: Comment results + if: github.event_name == 'workflow_dispatch' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cd ./php-version-benchmarks/tmp/php_${{ env.ID }} + NEWEST_RESULT_DIRECTORY=$(ls -td ${{ github.workspace }}/php-version-benchmarks/docs/results/${{ env.YEAR }}/*/ | head -1) + gh pr comment ${{ inputs.pull_request }} --body-file "${NEWEST_RESULT_DIRECTORY}result.md" --repo ${{ env.REPOSITORY }} - name: Cleanup if: always() run: |