quorum-code/.github/workflows/build.yml

name: Build Check
on:
  push:
    paths-ignore:
      - 'docs/**'
      - '**.md'
      - 'mkdocs.yml'
      - '.gitignore'
    branches:
      - master
env:
  GO_VERSION: 1.16
  GOPATH: ${{ github.workspace }}/go
  WORKING_DIR: ${{ github.workspace }}/go/src/github.com/ethereum/go-ethereum
jobs:
  build:
    name: 'Run tests and build on ${{ matrix.os }}'
    strategy:
      fail-fast: false
      matrix:
        # Not enable for macos as there's a consistent failure:
        # --- FAIL: TestUPNP_DDWRT (2.20s)
        # ###[error]    natupnp_test.go:165: not discovered
        # must be sommething with Github Actions VM networking setup.
        # Event Ubuntu requires a workaround
        os: [ "ubuntu-20.04" ]
    env:
      QUORUM_IGNORE_TEST_PACKAGES: github.com/ethereum/go-ethereum/les,github.com/ethereum/go-ethereum/les/flowcontrol,github.com/ethereum/go-ethereum/mobile
    runs-on: ${{ matrix.os }}
    steps:
      - name: 'Setup Go ${{ env.GO_VERSION }}'
        uses: actions/setup-go@v1
        with:
          go-version: ${{ env.GO_VERSION }}
      - name: 'Check out project files'
        uses: actions/checkout@v2
        with:
          submodules: recursive
          path: ${{ env.WORKING_DIR }}
      - name: 'Apply workaround to fix networking in Linux'
        if: runner.os == 'Linux'
        run: |
          # https://github.com/actions/virtual-environments/issues/798
          sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
      - name: 'Prepare environment'
        run: |
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
      - name: 'Run tests and build all'
        working-directory: ${{ env.WORKING_DIR }}
        run: |
          make test all
  docker-build:
    name: 'Build Docker image'
    runs-on: ubuntu-20.04
    steps:
      - name: 'Check out project files'
        uses: actions/checkout@v2
      - name: 'Build docker image'
        id: build
        run: |
          output_dir=${{ runner.temp }}/docker
          mkdir -p $output_dir
          docker build -t quorumengineering/quorum:pr .
          docker save quorumengineering/quorum:pr > quorum-pr.tar
          tar cfvz $output_dir/quorum-pr.tar.gz quorum-pr.tar
          echo "::set-output name=output_dir::$output_dir"
      - name: 'Upload workflow artifact - Docker image'
        uses: actions/upload-artifact@v1
        with:
          name: docker-image
          path: ${{ steps.build.outputs.output_dir }}
  acceptance-tests-basic:
    name: Acceptance tests (${{ matrix.tag }})
    needs:
      - docker-build
    if: success()
    strategy:
      fail-fast: false
      matrix:
        # list of tag expression being executed in parallel
        # for PR, only selective tests are run.
        # More comprehensive suites are scheduled to run in master
        tag:
          - 'basic || basic-raft || (advanced && raft) || networks/typical::raft'
          - 'basic || basic-istanbul || (advanced && istanbul) || empty-block-period || networks/typical::istanbul'
          - 'basic || basic-istanbul || (advanced && istanbul) || empty-block-period || block-reward || networks/typical::qbft'
          - 'gcmode && block-sync && networks/template::raft-3plus1'
          - 'gcmode && block-sync && networks/template::istanbul-3plus1'
          - 'gcmode && block-sync && networks/template::qbft-3plus1'
          - 'learner-peer-management || raftdnsenable && networks/template::raft-3plus1'
          - 'validator-management && networks/template::qbft-3plus1'
          - 'validator-management && networks/template::istanbul-3plus1'
          - 'hybrid-validator-management-manage-besu && networks/typical-hybrid::hybrid-template-q2b1'
          - 'hybrid-validator-management-manage-quorum && networks/typical-hybrid::hybrid-template-q1b2'
          - 'qbft-transition-network && networks/template::qbft-4nodes-transition'
          - 'basic || basic-raft || (advanced && raft) || networks/plugins::raft'
          - 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::qbft'
          - 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::istanbul'
          - 'basic || basic-raft || (advanced && raft) || networks/plugins::raft-account-plugin-hashicorp-vault'
          - 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::qbft-account-plugin-hashicorp-vault'
          - 'basic || basic-istanbul || (advanced && istanbul) || networks/plugins::istanbul-account-plugin-hashicorp-vault'
          - 'basic-rpc-security || networks/plugins::raft-rpc-security'
          - 'basic-rpc-security || networks/plugins::qbft-rpc-security'
          - 'basic-rpc-security || networks/plugins::istanbul-rpc-security'
          - 'migration && networks/template::raft-4nodes'
          - 'migration && networks/template::istanbul-4nodes'
          - 'migration && networks/template::raft-4nodes-ancientdb'
          - 'migration && networks/template::istanbul-4nodes-ancientdb'
          - 'permissions-v1 && networks/template::raft-3plus1'
          - 'permissions-v2 && networks/template::raft-3plus1'
          - 'privacy-enhancements-upgrade || networks/template::raft-4nodes-pe'
          - 'privacy-enhancements-upgrade || networks/template::istanbul-4nodes-pe'
          - 'multitenancy && networks/plugins::raft-multitenancy'
          - 'basic || basic-raft || (advanced && raft) || networks/typical::raft-simple-mps'
          - 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::qbft-simple-mps'
          - 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul-simple-mps'
          - 'basic || networks/typical::raftmps'
          - 'basic || networks/typical::qbftmps'
          - 'basic || networks/typical::istanbulmps'
          - 'mps-upgrade-txtrace || networks/template::raft-4nodes-mps'
          - 'mps-upgrade-txtrace || networks/template::istanbul-4nodes-mps'
          - 'mps-mixed-network-psr-check || networks/template::raft-4nodes-mps-mixed'
          - 'mps-mixed-network-psr-check || networks/template::istanbul-4nodes-mps-mixed'
          - '(basic && !nosupport && !mps && !(spam && !raw) && !eth-api-signed && !privacy-enhancements-disabled && !graphql && !async && !extension && !storage-root && !personal-api-signed) || networks/typical-hybrid::hybrid'
    runs-on: ubuntu-20.04
    steps:
      - name: 'Download workflow artifact - Docker image'
        uses: actions/download-artifact@v1
        with:
          name: docker-image
      - name: 'Load Docker image'
        id: setup
        run: |
          tar xfvz docker-image/quorum-pr.tar.gz
          docker load --input quorum-pr.tar
          docker_env_file="${{ runner.temp }}/env.list"
          echo "TF_VAR_quorum_docker_image={ name = \"quorumengineering/quorum:pr\", local = true }" >> $docker_env_file
          echo "::set-output name=outputDir::${{ runner.temp }}"
          echo "::set-output name=dockerEnvFile::$docker_env_file"
      - name: 'Run acceptance tests'
        run: |
          cat ${{ steps.setup.outputs.dockerEnvFile }}
          docker run --rm \
            --network host \
            -v /var/run/docker.sock:/var/run/docker.sock \
            -v ${{ steps.setup.outputs.outputDir }}:${{ steps.setup.outputs.outputDir  }} \
            --env-file ${{ steps.setup.outputs.dockerEnvFile }} \
            quorumengineering/acctests:latest test \
              -Pauto \
              -Dauto.outputDir=${{ steps.setup.outputs.outputDir  }} \
              -Dtags="${{ matrix.tag }}"
      - name: 'Debug'
        run: |
          docker images
          docker ps -a
  acceptance-tests-extra:
    name: Acceptance tests (${{ matrix.tag }})
    needs:
      - docker-build
    if: success()
    strategy:
      fail-fast: false
      matrix:
        # list of tag expression being executed in parallel
        include:
          # privacy enhancements tests
          - tag: '(basic && !privacy-enhancements-disabled) || privacy-enhancements || mandatory-recipients || basic-raft || (advanced && raft) || networks/typical::raft'
            privacy-enhancements: true
            privacy-precompile: false
            privacy-marker-transactions: false
          - tag: '(basic && !privacy-enhancements-disabled) || privacy-enhancements || mandatory-recipients || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul'
            privacy-enhancements: true
            privacy-precompile: false
            privacy-marker-transactions: false
          # privacy precompile/privacy marker transaction tests
          - tag: 'basic || basic-raft || (advanced && raft) || networks/typical::raft'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: false
          - tag: 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: false
          - tag: 'basic || basic-istanbul || (advanced && istanbul) || networks/typical::qbft'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: false
          - tag: '(multitenancy || privacy-precompile-enabled) && networks/plugins::raft-multitenancy'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: true
          - tag: '(basic && !privacy-precompile-disabled) || basic-raft || (advanced && raft) || networks/typical::raft-simple-mps'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: true
          - tag: '(basic && !privacy-precompile-disabled) || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul-simple-mps'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: true
          - tag: '(basic && !privacy-precompile-disabled) || basic-istanbul || (advanced && istanbul) || networks/typical::qbft-simple-mps'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: true
          - tag: '(basic && !privacy-precompile-disabled) || networks/typical::raftmps'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: true
          - tag: '(basic && !privacy-precompile-disabled) || networks/typical::istanbulmps'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: true
          - tag: '(basic && !privacy-precompile-disabled) || networks/typical::qbftmps'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: true
          # privacy enhancements + privacy precompile/privacy marker transaction tests
          - tag: '(basic && !privacy-enhancements-disabled && !privacy-precompile-disabled) || privacy-enhancements || mandatory-recipients || privacy-precompile-enabled || basic-raft || (advanced && raft) || networks/typical::raft'
            privacy-enhancements: true
            privacy-precompile: true
            privacy-marker-transactions: true
          - tag: '(basic && !privacy-enhancements-disabled && !privacy-precompile-disabled) || privacy-enhancements || mandatory-recipients || privacy-precompile-enabled || basic-istanbul || (advanced && istanbul) || networks/typical::istanbul'
            privacy-enhancements: true
            privacy-precompile: true
            privacy-marker-transactions: true
          - tag: '(basic && !privacy-enhancements-disabled && !privacy-precompile-disabled) || privacy-enhancements || mandatory-recipients || privacy-precompile-enabled || basic-istanbul || (advanced && istanbul) || networks/typical::qbft'
            privacy-enhancements: true
            privacy-precompile: true
            privacy-marker-transactions: true
          - tag: 'privacy-precompile-compatibility && networks/template::raft-4nodes'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: false # do not enable pmts as the test will do this on the necessary nodes
          - tag: 'privacy-precompile-compatibility && networks/template::istanbul-4nodes'
            privacy-enhancements: false
            privacy-precompile: true
            privacy-marker-transactions: false # do not enable pmts as the test will do this on the necessary nodes
    runs-on: ubuntu-20.04
    steps:
      - name: 'Download workflow artifact - Docker image'
        uses: actions/download-artifact@v1
        with:
          name: docker-image
      - name: 'Load Docker image'
        id: setup
        run: |
          tar xfvz docker-image/quorum-pr.tar.gz
          docker load --input quorum-pr.tar
          docker_env_file="${{ runner.temp }}/env.list"
          echo "TF_VAR_quorum_docker_image={ name = \"quorumengineering/quorum:pr\", local = true }" >> $docker_env_file
          echo "TF_VAR_privacy_enhancements={block=0, enabled=${{ matrix.privacy-enhancements}}}" >> $docker_env_file
          echo "TF_VAR_privacy_precompile={block=0, enabled=${{ matrix.privacy-precompile}}}" >> $docker_env_file
          echo "TF_VAR_privacy_marker_transactions=${{ matrix.privacy-marker-transactions}}" >> $docker_env_file
          echo "::set-output name=outputDir::${{ runner.temp }}"
          echo "::set-output name=dockerEnvFile::$docker_env_file"
      - name: 'Run extra acceptance tests'
        run: |
          cat ${{ steps.setup.outputs.dockerEnvFile }}
          docker run --rm \
            --network host \
            -v /var/run/docker.sock:/var/run/docker.sock \
            -v ${{ steps.setup.outputs.outputDir }}:${{ steps.setup.outputs.outputDir  }} \
            --env-file ${{ steps.setup.outputs.dockerEnvFile }} \
            quorumengineering/acctests:latest test \
              -Pauto \
              -Dauto.outputDir=${{ steps.setup.outputs.outputDir  }} \
              -Dtags="${{ matrix.tag }}"
      - name: 'Debug'
        run: |
          docker images
          docker ps -a

  peeps-tests:
    name: Run PEEPS tests
    needs:
      - docker-build
    runs-on: ubuntu-20.04
    steps:
      - name: 'Checkout'
        uses: actions/checkout@v2
      - name: 'Download workflow artifact - Docker image'
        uses: actions/download-artifact@v1
        with:
          name: docker-image
      - name: 'Load Docker image'
        id: setup
        run: |
          tar xfvz docker-image/quorum-pr.tar.gz
          docker load --input quorum-pr.tar
          docker image tag quorumengineering/quorum:pr quorumengineering/quorum:develop
          docker image ls
      - name: Set up Java
        uses: actions/setup-java@v2
        with:
          distribution: 'adopt'
          java-version: 11
          check-latest: true
      - name: PEEPS
        run: |
          cd build
          ./run-peeps.sh
      - name: PEEPS Test Report
        uses: mikepenz/action-junit-report@v2
        if: always()
        with:
          report_paths: '**/build/test-results/*/TEST-*.xml'
          check_name: PEEPS test report


  publish-docker:
    name: Publish Docker Image
    needs:
      - build
      - acceptance-tests-basic
      - acceptance-tests-extra
    runs-on: ubuntu-20.04
    steps:
      - name: 'Checkout'
        uses: actions/checkout@v2
      - name: 'Build and publish to Docker Hub'
        uses: docker/build-push-action@v1
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
          repository: ${{ secrets.DOCKER_REPO }}
          tags: develop
          add_git_labels: true
  notify:
    if: always()
    name: Notify
    needs:
      - build
      - publish-docker
    runs-on: ubuntu-20.04
    steps:
      - name: 'Setup metadata'
        id: setup
        run: |
          gitref_path="${{ github.ref }}"
          gitref_path=${gitref_path/refs\/heads/tree} # for refs/heads/my-branch
          gitref_path=${gitref_path/refs\/tags/tree}  # for refs/tags/v1.0.0
          gitref_path=${gitref_path#refs\/}           # for refs/pull/123/merge
          gitref_path=${gitref_path%/merge}           # for refs/pull/123/merge
          echo "::set-output name=gitref-path::$gitref_path"
      - name: 'Prepare Slack message with full info'
        id: status
        uses: actions/github-script@0.8.0
        with:
          script: |
            var gitref_path = "${{ steps.setup.outputs.gitref-path }}"
            ////////////////////////////////////
            // retrieve workflow run data
            ////////////////////////////////////
            console.log("get workflow run")
            const wf_run = await github.actions.getWorkflowRun({
                owner: context.repo.owner,
                repo: context.repo.repo,
                run_id: ${{ github.run_id }}
            })
            console.log(wf_run.data)
            console.log("get jobs for workflow run:", wf_run.data.jobs_url)
            const jobs_response = await github.request(wf_run.data.jobs_url)
            ////////////////////////////////////
            // build slack notification message
            ////////////////////////////////////
            // some utility functions
            var date_diff_func = function(start, end) {
                var duration = end - start
                // format the duration
                var delta = duration / 1000
                var days = Math.floor(delta / 86400)
                delta -= days * 86400
                var hours = Math.floor(delta / 3600) % 24
                delta -= hours * 3600
                var minutes = Math.floor(delta / 60) % 60
                delta -= minutes * 60
                var seconds = Math.floor(delta % 60)
                var format_func = function(v, text, check) {
                    if (v <= 0 && check) {
                        return ""
                    } else {
                        return v + text
                    }
                }
                return format_func(days, "d", true) + format_func(hours, "h", true) + format_func(minutes, "m", true) + format_func(seconds, "s", false)
            }
            var status_icon_func = function(s) {
                switch (s) {
                case "w_success":
                    return ":white_check_mark:"
                case "w_failure":
                    return ":no_entry:"
                case "w_cancelled":
                    return ":warning:"
                case "success":
                    return "\u2713"
                case "failure":
                    return "\u2717"
                default:
                    return "\u20e0"
                }
            }
            // build the message
            var job_blocks = []
            var is_wf_success = true
            var is_wf_failure = false
            for (j of jobs_response.data.jobs) {
                console.log(j.name, ":", j.status, j.conclusion, j.started_at, j.completed_at)
                // ignore the current job running this script
                if (j.status != "completed") {
                    continue
                }
                if (j.conclusion != "success") {
                  is_wf_success = false
                }
                if (j.conclusion == "failure") {
                  is_wf_failure = true
                }
                job_blocks.push({
                    type: "section",
                    text: {
                      type: "mrkdwn",
                      text: `${status_icon_func(j.conclusion)} <${j.html_url}|${j.name}> took ${date_diff_func(new Date(j.started_at), new Date(j.completed_at))}`
                    }
                })
            }
            var workflow_status = "w_cancelled"
            if (is_wf_success) {
              workflow_status = "w_success"
            } else if (is_wf_failure) {
              workflow_status = "w_failure"
            }
            var context_elements = [
              {
                  "type": "mrkdwn",
                  "text": "*Repo:* <https://github.com/${{ github.repository }}|${{ github.repository }}>"
              },
              {
                  "type": "mrkdwn",
                  "text": `*Branch:* <https://github.com/${{ github.repository }}/${gitref_path}|${{ github.ref }}>`
              },
              {
                  "type": "mrkdwn",
                  "text": `*Event:* ${wf_run.data.event}`
              },
              {
                  "type": "mrkdwn",
                  "text": `*Commit:* <https://github.com/${{ github.repository }}/commit/${wf_run.data.head_commit.id}|${wf_run.data.head_commit.id.substr(0, 8)}>`
              },
              {
                  "type": "mrkdwn",
                  "text": `*Author:* ${wf_run.data.head_commit.author.name}`
              }
            ]
            var header_blocks = [
                {
                    type: "section",
                    text: {
                        type: "mrkdwn",
                        text: `${status_icon_func(workflow_status)} *${{ github.workflow }}* <${wf_run.data.html_url}|#${{ github.run_number }}> took ${date_diff_func(new Date(wf_run.data.created_at), new Date(wf_run.data.updated_at))}`
                    }
                },
                {
                    type: "context",
                    elements: context_elements,
                },
                {
                    type: "divider"
                }
            ]
            var slack_msg = {
                blocks: [].concat(header_blocks, job_blocks)
            }
            return slack_msg
      - name: 'Prepare Slack message with partial info'
        id: short_status
        if: failure()
        uses: actions/github-script@0.8.0
        with:
          script: |
            ////////////////////////////////////
            // retrieve workflow run data
            ////////////////////////////////////
            const wf_run = await github.actions.getWorkflowRun({
                owner: context.repo.owner,
                repo: context.repo.repo,
                run_id: ${{ github.run_id }}
            })
            var date_diff_func = function(start, end) {
                var duration = end - start
                // format the duration
                var delta = duration / 1000
                var days = Math.floor(delta / 86400)
                delta -= days * 86400
                var hours = Math.floor(delta / 3600) % 24
                delta -= hours * 3600
                var minutes = Math.floor(delta / 60) % 60
                delta -= minutes * 60
                var seconds = Math.floor(delta % 60)
                var format_func = function(v, text, check) {
                    if (v <= 0 && check) {
                        return ""
                    } else {
                        return v + text
                    }
                }
                return format_func(days, "d", true) + format_func(hours, "h", true) + format_func(minutes, "m", true) + format_func(seconds, "s", false)
            }
            var slack_msg = {
                blocks: [
                  {
                      type: "section",
                      text: {
                          type: "mrkdwn",
                          text: `:skull_and_crossbones: *${{ github.workflow }}* <${wf_run.data.html_url}|#${{ github.run_number }}> (took ${date_diff_func(new Date(wf_run.data.created_at), new Date(wf_run.data.updated_at))})`
                      }
                  }
                ]
            }
            return slack_msg
      - name: 'Send to Slack'
        if: always()
        run: |
          cat <<JSON > long_message.json
          ${{ steps.status.outputs.result }}
          JSON
          cat <<JSON > short_message.json
          ${{ steps.short_status.outputs.result }}
          JSON
          _post() {
            curl -X POST ${{ secrets.SLACK_WEBHOOK_URL }} -H "Content-type: application/json" --data "@${1}"
          }
          _post "long_message.json" || _post "short_message.json"