diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5dfe7a1..d59437a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,7 +3,6 @@ name: "nightly" on: schedule: - cron: '0 0 * * *' - workflow_dispatch: env: HONEYCOMB_WRITEKEY: 7f3c63a70eecc61d635917de46bea4e6 @@ -15,35 +14,63 @@ jobs: runs-on: ubuntu-20.04 outputs: matrix: ${{ steps.get-matrix.outputs.matrix }} + steps: + - name: "Honeycomb: Start recording" + uses: kvrhdn/gha-buildevents@v1.0.2 + with: + apikey: ${{ env.HONEYCOMB_WRITEKEY }} + dataset: ${{ env.HONEYCOMB_DATASET }} + job-status: ${{ job.status }} + + - name: "Honeycomb: Start first step" + run: | + echo STEP_ID=0 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV + - name: Checkout Source uses: actions/checkout@v2 + if: ${{ github.repository_owner == 'puppetlabs' }} - name: Activate Ruby 2.7 uses: actions/setup-ruby@v1 + if: ${{ github.repository_owner == 'puppetlabs' }} with: ruby-version: "2.7" - name: Cache gems uses: actions/cache@v2 + if: ${{ github.repository_owner == 'puppetlabs' }} with: path: vendor/gems - key: ${{ runner.os }}-nightly-${{ hashFiles('**/Gemfile') }} + key: ${{ runner.os }}-${{ github.event_name }}-${{ hashFiles('**/Gemfile') }} restore-keys: | - ${{ runner.os }}-nightly- + ${{ runner.os }}-${{ github.event_name }}- ${{ runner.os }}- - name: Install gems + if: ${{ github.repository_owner == 'puppetlabs' }} run: | - bundle config path vendor/gems - bundle config jobs 8 - bundle config retry 3 - bundle install - bundle clean + buildevents cmd $TRACE_ID $STEP_ID 'bundle config path vendor/gems' -- bundle config path vendor/gems + buildevents cmd $TRACE_ID $STEP_ID 'bundle config jobs 8' -- bundle config jobs 8 + buildevents cmd $TRACE_ID $STEP_ID 'bundle config retry 3' -- bundle config retry 3 + buildevents cmd $TRACE_ID $STEP_ID 'bundle install' -- bundle install + buildevents cmd $TRACE_ID $STEP_ID 'bundle clean' -- bundle clean - name: Setup Acceptance Test Matrix id: get-matrix - run: "bundle exec matrix_from_metadata" + if: ${{ github.repository_owner == 'puppetlabs' }} + run: | + if [ '${{ github.repository_owner }}' == 'puppetlabs' ]; then + buildevents cmd $TRACE_ID $STEP_ID matrix_from_metadata -- bundle exec matrix_from_metadata + else + echo "::set-output name=matrix::{}" + fi + + - name: "Honeycomb: Record setup time" + if: ${{ always() }} + run: | + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Setup Test Matrix' Acceptance: needs: @@ -54,7 +81,27 @@ jobs: fail-fast: false matrix: ${{fromJson(needs.setup_matrix.outputs.matrix)}} + env: + BUILDEVENT_FILE: '../buildevents.txt' + steps: + - run: | + echo 'platform=${{ matrix.platform }}' >> $BUILDEVENT_FILE + echo 'collection=${{ matrix.collection }}' >> $BUILDEVENT_FILE + + - name: "Honeycomb: Start recording" + uses: kvrhdn/gha-buildevents@v1.0.2 + with: + apikey: ${{ env.HONEYCOMB_WRITEKEY }} + dataset: ${{ env.HONEYCOMB_DATASET }} + job-status: ${{ job.status }} + matrix-key: ${{ matrix.platform }}-${{ matrix.collection }} + + - name: "Honeycomb: start first step" + run: | + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-1 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV + - name: Checkout Source uses: actions/checkout@v2 @@ -67,26 +114,39 @@ jobs: uses: actions/cache@v2 with: path: vendor/gems - key: ${{ runner.os }}-nightly-${{ hashFiles('**/Gemfile') }} + key: ${{ runner.os }}-${{ github.event_name }}-${{ hashFiles('**/Gemfile') }} restore-keys: | - ${{ runner.os }}-nightly- + ${{ runner.os }}-${{ github.event_name }}- ${{ runner.os }}- - - name: Install gems + - name: "Honeycomb: Record cache setup time" + if: ${{ always() }} run: | - bundle config path vendor/gems - bundle config jobs 8 - bundle config retry 3 - bundle install - bundle clean + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Cache retrieval' + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-2 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV - - name: bundler environment + - name: Bundler Setup run: | - bundle env + buildevents cmd $TRACE_ID $STEP_ID 'bundle config path vendor/gems' -- bundle config path vendor/gems + buildevents cmd $TRACE_ID $STEP_ID 'bundle config jobs 8' -- bundle config jobs 8 + buildevents cmd $TRACE_ID $STEP_ID 'bundle config retry 3' -- bundle config retry 3 + buildevents cmd $TRACE_ID $STEP_ID 'bundle install' -- bundle install + buildevents cmd $TRACE_ID $STEP_ID 'bundle clean' -- bundle clean + echo ::group::bundler environment + buildevents cmd $TRACE_ID $STEP_ID 'bundle env' -- bundle env + echo ::endgroup:: + + - name: "Honeycomb: Record Bundler Setup time" + if: ${{ always() }} + run: | + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Bundler Setup' + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-3 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV - name: Provision test environment run: | - bundle exec rake 'litmus:provision[provision::provision_service,${{ matrix.platform }}]' + buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:provision ${{ matrix.platform }}' -- bundle exec rake 'litmus:provision[provision::provision_service,${{ matrix.platform }}]' echo ::group::=== REQUEST === cat request.json || true echo @@ -103,8 +163,8 @@ jobs: with: timeout_minutes: 30 max_attempts: 5 - retry_wait_seconds: 90 - command: bundle exec rake 'litmus:install_agent[${{ matrix.collection }}]' + retry_wait_seconds: 60 + command: buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:install_agent ${{ matrix.collection }}' -- bundle exec rake 'litmus:install_agent[${{ matrix.collection }}]' # The agent installer on windows does not finish in time for this to work. To # work around this for now, retry after a minute if installing the module failed. @@ -112,21 +172,45 @@ jobs: uses: nick-invision/retry@v1 with: timeout_minutes: 30 - max_attempts: 5 + max_attempts: 2 retry_wait_seconds: 60 - command: bundle exec rake 'litmus:install_module' + command: buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:install_module' -- bundle exec rake 'litmus:install_module' + + - name: "Honeycomb: Record deployment times" + if: ${{ always() }} + run: | + echo ::group::honeycomb step + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Deploy test system' + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-4 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV + echo ::endgroup:: - name: Run acceptance tests - run: bundle exec rake 'litmus:acceptance:parallel' + run: | + buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:acceptance:parallel' -- bundle exec rake 'litmus:acceptance:parallel' + + - name: "Honeycomb: Record acceptance testing times" + if: ${{ always() }} + run: | + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Run acceptance tests' + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-5 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV - name: Remove test environment if: ${{ always() }} run: | - bundle exec rake 'litmus:tear_down' - echo ::group::=== REQUEST === - cat request.json || true - echo - echo ::endgroup:: + if [ -f inventory.yaml ]; then + buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:tear_down' -- bundle exec rake 'litmus:tear_down' + echo ::group::=== REQUEST === + cat request.json || true + echo + echo ::endgroup:: + fi + + - name: "Honeycomb: Record removal times" + if: ${{ always() }} + run: | + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Remove test environment' slack-workflow-status: if: always() @@ -139,8 +223,8 @@ jobs: uses: Gamesight/slack-workflow-status@master with: # Required Input - repo_token: ${{secrets.GITHUB_TOKEN}} - slack_webhook_url: ${{secrets.SLACK_WEBHOOK}} + repo_token: ${{ secrets.GITHUB_TOKEN }} + slack_webhook_url: ${{ secrets.SLACK_WEBHOOK }} # Optional Input channel: '#team-ia-bots' name: 'GABot' diff --git a/.github/workflows/pr_test.yml b/.github/workflows/pr_test.yml index 7b47199..17f5a64 100644 --- a/.github/workflows/pr_test.yml +++ b/.github/workflows/pr_test.yml @@ -12,35 +12,63 @@ jobs: runs-on: ubuntu-20.04 outputs: matrix: ${{ steps.get-matrix.outputs.matrix }} + steps: + - name: "Honeycomb: Start recording" + uses: kvrhdn/gha-buildevents@v1.0.2 + with: + apikey: ${{ env.HONEYCOMB_WRITEKEY }} + dataset: ${{ env.HONEYCOMB_DATASET }} + job-status: ${{ job.status }} + + - name: "Honeycomb: Start first step" + run: | + echo STEP_ID=0 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV + - name: Checkout Source uses: actions/checkout@v2 + if: ${{ github.repository_owner == 'puppetlabs' }} - name: Activate Ruby 2.7 uses: actions/setup-ruby@v1 + if: ${{ github.repository_owner == 'puppetlabs' }} with: ruby-version: "2.7" - name: Cache gems uses: actions/cache@v2 + if: ${{ github.repository_owner == 'puppetlabs' }} with: path: vendor/gems - key: ${{ runner.os }}-pr-${{ hashFiles('**/Gemfile') }} + key: ${{ runner.os }}-${{ github.event_name }}-${{ hashFiles('**/Gemfile') }} restore-keys: | - ${{ runner.os }}-pr- + ${{ runner.os }}-${{ github.event_name }}- ${{ runner.os }}- - name: Install gems + if: ${{ github.repository_owner == 'puppetlabs' }} run: | - bundle config path vendor/gems - bundle config jobs 8 - bundle config retry 3 - bundle install - bundle clean + buildevents cmd $TRACE_ID $STEP_ID 'bundle config path vendor/gems' -- bundle config path vendor/gems + buildevents cmd $TRACE_ID $STEP_ID 'bundle config jobs 8' -- bundle config jobs 8 + buildevents cmd $TRACE_ID $STEP_ID 'bundle config retry 3' -- bundle config retry 3 + buildevents cmd $TRACE_ID $STEP_ID 'bundle install' -- bundle install + buildevents cmd $TRACE_ID $STEP_ID 'bundle clean' -- bundle clean - name: Setup Acceptance Test Matrix id: get-matrix - run: "bundle exec matrix_from_metadata" + if: ${{ github.repository_owner == 'puppetlabs' }} + run: | + if [ '${{ github.repository_owner }}' == 'puppetlabs' ]; then + buildevents cmd $TRACE_ID $STEP_ID matrix_from_metadata -- bundle exec matrix_from_metadata + else + echo "::set-output name=matrix::{}" + fi + + - name: "Honeycomb: Record setup time" + if: ${{ always() }} + run: | + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Setup Test Matrix' Acceptance: needs: @@ -51,7 +79,27 @@ jobs: fail-fast: false matrix: ${{fromJson(needs.setup_matrix.outputs.matrix)}} + env: + BUILDEVENT_FILE: '../buildevents.txt' + steps: + - run: | + echo 'platform=${{ matrix.platform }}' >> $BUILDEVENT_FILE + echo 'collection=${{ matrix.collection }}' >> $BUILDEVENT_FILE + + - name: "Honeycomb: Start recording" + uses: kvrhdn/gha-buildevents@v1.0.2 + with: + apikey: ${{ env.HONEYCOMB_WRITEKEY }} + dataset: ${{ env.HONEYCOMB_DATASET }} + job-status: ${{ job.status }} + matrix-key: ${{ matrix.platform }}-${{ matrix.collection }} + + - name: "Honeycomb: start first step" + run: | + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-1 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV + - name: Checkout Source uses: actions/checkout@v2 @@ -64,26 +112,39 @@ jobs: uses: actions/cache@v2 with: path: vendor/gems - key: ${{ runner.os }}-pr-${{ hashFiles('**/Gemfile') }} + key: ${{ runner.os }}-${{ github.event_name }}-${{ hashFiles('**/Gemfile') }} restore-keys: | - ${{ runner.os }}-pr- + ${{ runner.os }}-${{ github.event_name }}- ${{ runner.os }}- - - name: Install gems + - name: "Honeycomb: Record cache setup time" + if: ${{ always() }} run: | - bundle config path vendor/gems - bundle config jobs 8 - bundle config retry 3 - bundle install - bundle clean + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Cache retrieval' + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-2 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV - - name: bundler environment + - name: Bundler Setup run: | - bundle env + buildevents cmd $TRACE_ID $STEP_ID 'bundle config path vendor/gems' -- bundle config path vendor/gems + buildevents cmd $TRACE_ID $STEP_ID 'bundle config jobs 8' -- bundle config jobs 8 + buildevents cmd $TRACE_ID $STEP_ID 'bundle config retry 3' -- bundle config retry 3 + buildevents cmd $TRACE_ID $STEP_ID 'bundle install' -- bundle install + buildevents cmd $TRACE_ID $STEP_ID 'bundle clean' -- bundle clean + echo ::group::bundler environment + buildevents cmd $TRACE_ID $STEP_ID 'bundle env' -- bundle env + echo ::endgroup:: + + - name: "Honeycomb: Record Bundler Setup time" + if: ${{ always() }} + run: | + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Bundler Setup' + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-3 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV - name: Provision test environment run: | - bundle exec rake 'litmus:provision[provision::provision_service,${{ matrix.platform }}]' + buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:provision ${{ matrix.platform }}' -- bundle exec rake 'litmus:provision[provision::provision_service,${{ matrix.platform }}]' echo ::group::=== REQUEST === cat request.json || true echo @@ -100,8 +161,8 @@ jobs: with: timeout_minutes: 30 max_attempts: 5 - retry_wait_seconds: 90 - command: bundle exec rake 'litmus:install_agent[${{ matrix.collection }}]' + retry_wait_seconds: 60 + command: buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:install_agent ${{ matrix.collection }}' -- bundle exec rake 'litmus:install_agent[${{ matrix.collection }}]' # The agent installer on windows does not finish in time for this to work. To # work around this for now, retry after a minute if installing the module failed. @@ -109,18 +170,42 @@ jobs: uses: nick-invision/retry@v1 with: timeout_minutes: 30 - max_attempts: 5 + max_attempts: 2 retry_wait_seconds: 60 - command: bundle exec rake 'litmus:install_module' + command: buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:install_module' -- bundle exec rake 'litmus:install_module' + + - name: "Honeycomb: Record deployment times" + if: ${{ always() }} + run: | + echo ::group::honeycomb step + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Deploy test system' + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-4 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV + echo ::endgroup:: - name: Run acceptance tests - run: bundle exec rake 'litmus:acceptance:parallel' + run: | + buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:acceptance:parallel' -- bundle exec rake 'litmus:acceptance:parallel' + + - name: "Honeycomb: Record acceptance testing times" + if: ${{ always() }} + run: | + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Run acceptance tests' + echo STEP_ID=${{ matrix.platform }}-${{ matrix.collection }}-5 >> $GITHUB_ENV + echo STEP_START=$(date +%s) >> $GITHUB_ENV - name: Remove test environment if: ${{ always() }} run: | - bundle exec rake 'litmus:tear_down' - echo ::group::=== REQUEST === - cat request.json || true - echo - echo ::endgroup:: + if [ -f inventory.yaml ]; then + buildevents cmd $TRACE_ID $STEP_ID 'rake litmus:tear_down' -- bundle exec rake 'litmus:tear_down' + echo ::group::=== REQUEST === + cat request.json || true + echo + echo ::endgroup:: + fi + + - name: "Honeycomb: Record removal times" + if: ${{ always() }} + run: | + buildevents step $TRACE_ID $STEP_ID $STEP_START 'Remove test environment' diff --git a/.sync.yml b/.sync.yml index 0458476..2c0f5e1 100644 --- a/.sync.yml +++ b/.sync.yml @@ -54,3 +54,7 @@ spec/spec_helper.rb: unmanaged: false .gitpod.yml: unmanaged: false +.github/workflows/nightly.yml: + unmanaged: false +.github/workflows/pr_test.yml: + unmanaged: false diff --git a/metadata.json b/metadata.json index d1d4aa8..480d445 100644 --- a/metadata.json +++ b/metadata.json @@ -82,6 +82,6 @@ } ], "template-url": "https://github.com/puppetlabs/pdk-templates#main", - "template-ref": "heads/main-0-g874030e", + "template-ref": "heads/main-0-g5afcd3d", "pdk-version": "1.18.1" }