From a0909df8683284bf86f69bf34456a743ee8a343c Mon Sep 17 00:00:00 2001 From: hjpotter92 Date: Tue, 6 Aug 2024 15:55:45 +0530 Subject: [PATCH 1/9] workflows: Update build workflow to not rely on curl based upload --- .github/workflows/build.yaml | 59 ++++++++++++++++++---------------- .github/workflows/release.yaml | 2 +- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6e93599485..f3e606b9ae 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -15,7 +15,7 @@ concurrency: jobs: linux-build: name: Build binaries for ${{ matrix.target.GOOS }}-${{ matrix.target.type }}-${{ matrix.target.GOARCH }} - runs-on: ${{ matrix.target.runner }} + runs-on: ubuntu-20.04 container: image: ${{ matrix.target.container }} env: @@ -26,31 +26,26 @@ jobs: target: - GOOS: linux GOARCH: amd64 - runner: ubuntu-20.04 container: ubuntu:20.04 type: cpu - GOOS: linux GOARCH: arm64 - runner: ubuntu-20.04 container: ubuntu:20.04 type: cpu - GOOS: linux GOARCH: amd64 - runner: ubuntu-20.04 container: livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04 type: gpu - GOOS: linux GOARCH: arm64 - runner: ubuntu-20.04 container: livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04 type: gpu - GOOS: windows GOARCH: amd64 - runner: ubuntu-20.04 container: ubuntu:22.04 type: cpu @@ -91,6 +86,7 @@ jobs: echo "GOOS=${{ matrix.target.GOOS }}" >> $GITHUB_ENV echo "GO_BUILD_DIR=lp-builds/" >> $GITHUB_ENV echo "PKG_CONFIG_PATH=/github/home/compiled/lib/pkgconfig" >> $GITHUB_ENV + mkdir -p "lp-builds/livepeer-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}" releases/ - name: Set GPU build environment if: matrix.target.type == 'gpu' @@ -130,24 +126,26 @@ jobs: git config --global --add safe.directory '*' ./ci_env.sh make - - name: Upload build - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository - env: - GHA_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} - GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }} - GCLOUD_SECRET: ${{ secrets.GCLOUD_SECRET }} - DISCORD_URL: ${{ secrets.DISCORD_URL }} - run: ./upload_build.sh + - name: Archive binaries for windows + if: matrix.target.GOOS == 'windows' + run: | + cd "$GO_BUILD_DIR/" + find . -type f -exec mv '{}' 'livepeer-${GOOS}-${GOARCH}/{}' \; + zip -9rq "../releases/livepeer-${GOOS}-${GOARCH}.zip" ./ + + - name: Archive binaries + if: matrix.platform.name != 'windows' + run: | + cd "$GO_BUILD_DIR/" + find . -type f -exec mv '{}' 'livepeer-${GOOS}-${GOARCH}/{}' \; + tar -czvf "../releases/livepeer-${GOOS}-${GOARCH}.tar.gz" ./ - name: Upload artifacts for cutting release - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: release-artifacts + name: release-artifacts-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }} path: releases/ - - name: Notify new build upload - run: curl -X POST https://holy-bread-207a.livepeer.workers.dev - macos-build: name: Build binaries for ${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }} runs-on: ${{ matrix.target.runner }} @@ -161,7 +159,7 @@ jobs: - GOOS: darwin GOARCH: arm64 - runner: macos-latest + runner: macos-14-xlarge steps: - name: Check out code @@ -230,6 +228,12 @@ jobs: app-notarization-team-id: ${{ secrets.CI_MACOS_NOTARIZATION_TEAM_ID }} binary-path: "lp-builds/" + - name: Archive binaries + if: matrix.platform.name != 'windows' + run: | + cd "$GO_BUILD_DIR/" + tar -czvf "../releases/livepeer-${GOOS}-${GOARCH}.tar.gz" . + - name: Upload build if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository env: @@ -240,14 +244,11 @@ jobs: run: ./upload_build.sh - name: Upload artifacts for cutting release - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: release-artifacts + name: release-artifacts-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }} path: releases/ - - name: Notify new build upload - run: curl -X POST https://holy-bread-207a.livepeer.workers.dev - upload: name: Upload artifacts to google bucket permissions: @@ -259,10 +260,11 @@ jobs: - linux-build steps: - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: release-artifacts + pattern: release-artifacts-* path: releases/ + merge-multiple: true - name: Generate sha256 checksum and gpg signatures for release artifacts uses: livepeer/action-gh-checksum-and-gpg-sign@latest @@ -302,3 +304,6 @@ jobs: path: ${{ steps.branch-manifest.outputs.manifest-file }} destination: "build.livepeer.live/${{ github.event.repository.name }}/" parent: false + + - name: Notify new build upload + run: curl -X POST https://holy-bread-207a.livepeer.workers.dev diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6d7bab98e0..270c698e61 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -41,7 +41,7 @@ jobs: gpg-key-passphrase: ${{ secrets.CI_GPG_SIGNING_PASSPHRASE }} - name: Release to github - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 if: ${{ steps.match-tag.outputs.match != '' }} with: generate_release_notes: true From 817c1c261f2bdd06e44ccc429be375b42d4e32ec Mon Sep 17 00:00:00 2001 From: hjpotter92 Date: Tue, 6 Aug 2024 16:22:24 +0530 Subject: [PATCH 2/9] build.yaml: Use release tag for separation of built artifacts --- .github/workflows/build.yaml | 31 +++++++++++++++---------------- .github/workflows/release.yaml | 6 ++++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f3e606b9ae..93e29f5cb7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -86,7 +86,8 @@ jobs: echo "GOOS=${{ matrix.target.GOOS }}" >> $GITHUB_ENV echo "GO_BUILD_DIR=lp-builds/" >> $GITHUB_ENV echo "PKG_CONFIG_PATH=/github/home/compiled/lib/pkgconfig" >> $GITHUB_ENV - mkdir -p "lp-builds/livepeer-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}" releases/ + echo "LP_BUILD_DIR=livepeer-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}" >> $GITHUB_ENV + mkdir -p lp-builds/ releases/ - name: Set GPU build environment if: matrix.target.type == 'gpu' @@ -95,6 +96,7 @@ jobs: echo "LIBRARY_PATH=/usr/local/cuda_${{ matrix.target.GOARCH }}/lib64" >> $GITHUB_ENV echo "CGO_LDFLAGS=-L/usr/local/cuda_${{ matrix.target.GOARCH }}/lib64" >> $GITHUB_ENV echo "RELEASE_TAG=gpu" >> $GITHUB_ENV + echo "LP_BUILD_DIR=livepeer-${{ matrix.target.GOOS }}-gpu-${{ matrix.target.GOARCH }}" >> $GITHUB_ENV - name: Install dependencies run: | @@ -129,21 +131,23 @@ jobs: - name: Archive binaries for windows if: matrix.target.GOOS == 'windows' run: | + mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/" cd "$GO_BUILD_DIR/" - find . -type f -exec mv '{}' 'livepeer-${GOOS}-${GOARCH}/{}' \; - zip -9rq "../releases/livepeer-${GOOS}-${GOARCH}.zip" ./ + find . -type f -exec mv '{}' "$LP_BUILD_DIR/" \; + zip -9rq "../releases/$LP_BUILD_DIR.zip" ./ - name: Archive binaries if: matrix.platform.name != 'windows' run: | + mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/" cd "$GO_BUILD_DIR/" - find . -type f -exec mv '{}' 'livepeer-${GOOS}-${GOARCH}/{}' \; - tar -czvf "../releases/livepeer-${GOOS}-${GOARCH}.tar.gz" ./ + find . -type f -exec mv '{}' "$LP_BUILD_DIR" \; + tar -czvf "../releases/$LP_BUILD_DIR.tar.gz" ./ - name: Upload artifacts for cutting release uses: actions/upload-artifact@v4 with: - name: release-artifacts-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }} + name: release-artifacts-${{ matrix.target.GOOS }}-${{ matrix.target.type }}-${{ matrix.target.GOARCH }} path: releases/ macos-build: @@ -183,6 +187,8 @@ jobs: echo "GOARCH=${{ matrix.target.GOARCH }}" >> $GITHUB_ENV echo "GOOS=${{ matrix.target.GOOS }}" >> $GITHUB_ENV echo "GO_BUILD_DIR=lp-builds/" >> $GITHUB_ENV + echo "LP_BUILD_DIR=livepeer-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}" >> $GITHUB_ENV + mkdir -p lp-builds/ releases/ - name: Cache ffmpeg id: cache-ffmpeg @@ -231,17 +237,10 @@ jobs: - name: Archive binaries if: matrix.platform.name != 'windows' run: | + mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/" cd "$GO_BUILD_DIR/" - tar -czvf "../releases/livepeer-${GOOS}-${GOARCH}.tar.gz" . - - - name: Upload build - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository - env: - GHA_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} - GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }} - GCLOUD_SECRET: ${{ secrets.GCLOUD_SECRET }} - DISCORD_URL: ${{ secrets.DISCORD_URL }} - run: ./upload_build.sh + find . -type f -exec mv '{}' "$LP_BUILD_DIR" \; + tar -czvf "../releases/${LP_BUILD_DIR}.tar.gz" . - name: Upload artifacts for cutting release uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 270c698e61..288685d413 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,10 +19,12 @@ jobs: ref: ${{ github.event.workflow_run.head_branch }} - name: Download artifacts from build stage - uses: dawidd6/action-download-artifact@v3 + uses: dawidd6/action-download-artifact@v6 with: workflow: build.yaml - name: release-artifacts + run_id: ${{ github.event.workflow_run.id }} + name: release-artifacts-.* + name_is_regexp: true path: releases/ - uses: actions-ecosystem/action-regex-match@v2 From 3f048d052147045bb684cc1258af431fb01e364d Mon Sep 17 00:00:00 2001 From: hjpotter92 Date: Tue, 6 Aug 2024 16:55:03 +0530 Subject: [PATCH 3/9] build.yaml: Fix cache paths for ffmpeg install step --- .github/workflows/build.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 93e29f5cb7..2e88cf27f2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -77,7 +77,7 @@ jobs: id: cache-ffmpeg uses: actions/cache@v3 with: - path: /home/runner/compiled + path: /github/home/compiled key: ${{ runner.os }}-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}-${{ matrix.target.type }}-ffmpeg-${{ hashFiles('**/install_ffmpeg.sh') }} - name: Set build environment @@ -290,19 +290,21 @@ jobs: - name: Upload release archives to Google Cloud id: upload-archives - uses: google-github-actions/upload-cloud-storage@v1 + uses: google-github-actions/upload-cloud-storage@v2 with: path: "releases" destination: "build.livepeer.live/${{ github.event.repository.name }}/${{ (github.ref_type == 'tag' && github.ref_name) || github.sha }}" parent: false + process_gcloudignore: false - name: Upload branch manifest file id: upload-manifest - uses: google-github-actions/upload-cloud-storage@v1 + uses: google-github-actions/upload-cloud-storage@v2 with: path: ${{ steps.branch-manifest.outputs.manifest-file }} destination: "build.livepeer.live/${{ github.event.repository.name }}/" parent: false + process_gcloudignore: false - name: Notify new build upload run: curl -X POST https://holy-bread-207a.livepeer.workers.dev From 23ed33f9cd74e371deee3a461cf73c7b8f3966e6 Mon Sep 17 00:00:00 2001 From: hjpotter92 Date: Wed, 7 Aug 2024 15:30:39 +0530 Subject: [PATCH 4/9] build.yaml: Add step to generate discord embeddable message --- .github/discord-embed-webhook.py | 85 ++++++++++++++++++++++++++++++++ .github/workflows/build.yaml | 19 +++++++ 2 files changed, 104 insertions(+) create mode 100644 .github/discord-embed-webhook.py diff --git a/.github/discord-embed-webhook.py b/.github/discord-embed-webhook.py new file mode 100644 index 0000000000..5e9a679059 --- /dev/null +++ b/.github/discord-embed-webhook.py @@ -0,0 +1,85 @@ +import argparse +import os +import json +import pathlib +from typing import Any +import urllib + + +GITHUB_CONTEXT_JSON = os.environ.get("GITHUB_CONTEXT_JSON", "{}") +DOWNLOAD_BASE_URL_TEMPLATE = ( + "https://build.livepeer.live/go-livepeer/{version}/{filename}" +) + + +def get_embeds(ref_name: str, checksums: list[str]) -> list[dict[Any, Any]]: + embeds = [] + for line in checksums: + shasum, filename = line.split() + download_url = DOWNLOAD_BASE_URL_TEMPLATE.format( + version=ref_name, + filename=filename, + ) + title = filename[9:].split(".")[0] + parts = title.split("-") + platform, architecture = parts[0], parts[1] + if len(parts) == 3: + architecture = parts[2] + embeds.append( + { + "title": title, + "url": download_url, + "color": 2928914, + "fields": [ + {"name": "platform", "value": platform, "inline": True}, + {"name": "architecture", "value": architecture, "inline": True}, + {"name": "filename", "value": filename, "inline": True}, + { + "name": "Download URL", + "value": download_url, + }, + { + "name": "SHA256 checksum", + "value": f"`{shasum}`", + }, + ], + }, + ) + return embeds + + +def main(args): + checksums = [] + github_context = json.loads(GITHUB_CONTEXT_JSON) + head = github_context.get("head", {}) + user = head.get("user", {}) + repo = head.get("repo", {}) + releases_dir = pathlib.Path("releases/") + for f in releases_dir.iterdir(): + print(f) + if f.suffix != ".txt": + continue + checksums = f.read_text().splitlines() + print(checksums) + content = { + "content": ":white_check_mark: Build succeeded for go-livepeer :white_check_mark: \n\n" + f"Branch: `{head.get('ref')}`\n" + f'Latest commit: "{args.git_commit}" by **{args.git_committer}**\n\n' + f"Commit SHA: (`{head.get('sha')}`)[{repo.get('commits_url')}]", + "embed": get_embeds(args.ref_name, checksums), + "username": user.get("name"), + "avatar_url": user.get("avatar_url"), + "flags": 4, + } + print(json.dumps(content)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Discord embed content generator for build system") + parser.add_argument("--discord-url", help="Discord webhook URL") + parser.add_argument("--ref-name", help="Tag/branch/commit for current build") + parser.add_argument("--git-commit", help="git commit message") + parser.add_argument("--git-committer", help="git commit author name") + args = parser.parse_args() + print(args) + main(args) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2e88cf27f2..84d0ad10b3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -258,6 +258,14 @@ jobs: - macos-build - linux-build steps: + - name: Check out code + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + # Check https://github.com/livepeer/go-livepeer/pull/1891 + # for ref value discussion + ref: ${{ github.event.pull_request.head.sha }} + - name: Download artifacts uses: actions/download-artifact@v4 with: @@ -306,5 +314,16 @@ jobs: parent: false process_gcloudignore: false + - name: Trigger discord webhook + shell: bash + env: + GITHUB_CONTEXT_JSON: ${{ toJson(github.event.pull_request) }} + run: | + python .github/discord-embed-webhook.py \ + --ref-name="${{ (github.ref_type == 'tag' && github.ref_name) || github.sha }}" + --discord-url="${{ secrets.DISCORD_URL }}" + --git-commit="$(git log -1 --pretty=format:'%s')" + --git-committer="$(git log -1 --pretty=format:'%an')" + - name: Notify new build upload run: curl -X POST https://holy-bread-207a.livepeer.workers.dev From f14a8fbdc74774d8492b98252e405ba5621a607e Mon Sep 17 00:00:00 2001 From: hjpotter92 Date: Wed, 7 Aug 2024 15:34:43 +0530 Subject: [PATCH 5/9] build.yaml: Caching ffmpeg install step leads to failure. --- .github/discord-embed-webhook.py | 1 - .github/workflows/build.yaml | 14 +++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/discord-embed-webhook.py b/.github/discord-embed-webhook.py index 5e9a679059..4161248dcd 100644 --- a/.github/discord-embed-webhook.py +++ b/.github/discord-embed-webhook.py @@ -81,5 +81,4 @@ def main(args): parser.add_argument("--git-commit", help="git commit message") parser.add_argument("--git-committer", help="git commit author name") args = parser.parse_args() - print(args) main(args) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 84d0ad10b3..98e06e028f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -73,13 +73,6 @@ jobs: cache: true cache-dependency-path: go.sum - - name: Cache ffmpeg - id: cache-ffmpeg - uses: actions/cache@v3 - with: - path: /github/home/compiled - key: ${{ runner.os }}-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}-${{ matrix.target.type }}-ffmpeg-${{ hashFiles('**/install_ffmpeg.sh') }} - - name: Set build environment run: | echo "GOARCH=${{ matrix.target.GOARCH }}" >> $GITHUB_ENV @@ -118,7 +111,6 @@ jobs: run: go mod download - name: Install ffmpeg - if: steps.cache-ffmpeg.outputs.cache-hit != 'true' run: ./install_ffmpeg.sh - name: Build binaries @@ -320,9 +312,9 @@ jobs: GITHUB_CONTEXT_JSON: ${{ toJson(github.event.pull_request) }} run: | python .github/discord-embed-webhook.py \ - --ref-name="${{ (github.ref_type == 'tag' && github.ref_name) || github.sha }}" - --discord-url="${{ secrets.DISCORD_URL }}" - --git-commit="$(git log -1 --pretty=format:'%s')" + --ref-name="${{ (github.ref_type == 'tag' && github.ref_name) || github.sha }}" \ + --discord-url="${{ secrets.DISCORD_URL }}" \ + --git-commit="$(git log -1 --pretty=format:'%s')" \ --git-committer="$(git log -1 --pretty=format:'%an')" - name: Notify new build upload From 8a7fe88f4b767183a87ce2700e051b46b747a5da Mon Sep 17 00:00:00 2001 From: hjpotter92 Date: Wed, 7 Aug 2024 17:25:45 +0530 Subject: [PATCH 6/9] embed-webhook: Trigger webhook with generated payload --- .github/discord-embed-webhook.py | 75 ++++++++++++-------------------- .github/workflows/build.yaml | 3 +- 2 files changed, 29 insertions(+), 49 deletions(-) diff --git a/.github/discord-embed-webhook.py b/.github/discord-embed-webhook.py index 4161248dcd..58a39dc5d9 100644 --- a/.github/discord-embed-webhook.py +++ b/.github/discord-embed-webhook.py @@ -2,8 +2,8 @@ import os import json import pathlib -from typing import Any -import urllib + +from discord_webhook import DiscordEmbed, DiscordWebhook GITHUB_CONTEXT_JSON = os.environ.get("GITHUB_CONTEXT_JSON", "{}") @@ -12,8 +12,7 @@ ) -def get_embeds(ref_name: str, checksums: list[str]) -> list[dict[Any, Any]]: - embeds = [] +def get_embeds(embed: DiscordEmbed, ref_name: str, checksums: list[str]): for line in checksums: shasum, filename = line.split() download_url = DOWNLOAD_BASE_URL_TEMPLATE.format( @@ -21,57 +20,37 @@ def get_embeds(ref_name: str, checksums: list[str]) -> list[dict[Any, Any]]: filename=filename, ) title = filename[9:].split(".")[0] - parts = title.split("-") - platform, architecture = parts[0], parts[1] - if len(parts) == 3: - architecture = parts[2] - embeds.append( - { - "title": title, - "url": download_url, - "color": 2928914, - "fields": [ - {"name": "platform", "value": platform, "inline": True}, - {"name": "architecture", "value": architecture, "inline": True}, - {"name": "filename", "value": filename, "inline": True}, - { - "name": "Download URL", - "value": download_url, - }, - { - "name": "SHA256 checksum", - "value": f"`{shasum}`", - }, - ], - }, - ) - return embeds + embed.add_embed_field(name=title, value=download_url, inline=False) + embed.add_embed_field(name="SHA256 checksum", value=shasum, inline=False) def main(args): checksums = [] github_context = json.loads(GITHUB_CONTEXT_JSON) head = github_context.get("head", {}) - user = head.get("user", {}) repo = head.get("repo", {}) - releases_dir = pathlib.Path("releases/") - for f in releases_dir.iterdir(): - print(f) - if f.suffix != ".txt": - continue - checksums = f.read_text().splitlines() - print(checksums) - content = { - "content": ":white_check_mark: Build succeeded for go-livepeer :white_check_mark: \n\n" - f"Branch: `{head.get('ref')}`\n" - f'Latest commit: "{args.git_commit}" by **{args.git_committer}**\n\n' - f"Commit SHA: (`{head.get('sha')}`)[{repo.get('commits_url')}]", - "embed": get_embeds(args.ref_name, checksums), - "username": user.get("name"), - "avatar_url": user.get("avatar_url"), - "flags": 4, - } - print(json.dumps(content)) + checksums_file = pathlib.Path("releases") / f"{args.ref_name}_checksums.txt" + commit_url = f'{repo.get("html_url")}/commit/{head.get("sha")}' + checksums = checksums_file.read_text().splitlines() + webhook = DiscordWebhook( + url=args.discord_url, + content=":white_check_mark: Build succeeded for go-livepeer :white_check_mark:", + username="[BOT] Livepeer builder", + ) + webhook.add_file(filename=checksums_file.name, file=checksums_file.read_bytes()) + embed = DiscordEmbed( + title=head.get("ref"), + description=args.git_commit, + color=2928914, + url=commit_url, + ) + embed.add_embed_field(name="Commit SHA", value=head.get("sha")) + embed.set_author(name=args.git_committer) + get_embeds(embed, args.ref_name, checksums) + embed.set_timestamp() + webhook.add_embed(embed) + response = webhook.execute() + print(response) if __name__ == "__main__": diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 98e06e028f..83d4c9432c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -129,7 +129,7 @@ jobs: zip -9rq "../releases/$LP_BUILD_DIR.zip" ./ - name: Archive binaries - if: matrix.platform.name != 'windows' + if: matrix.target.GOOS != 'windows' run: | mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/" cd "$GO_BUILD_DIR/" @@ -311,6 +311,7 @@ jobs: env: GITHUB_CONTEXT_JSON: ${{ toJson(github.event.pull_request) }} run: | + pip install -U discord-webhook --no-cache-dir python .github/discord-embed-webhook.py \ --ref-name="${{ (github.ref_type == 'tag' && github.ref_name) || github.sha }}" \ --discord-url="${{ secrets.DISCORD_URL }}" \ From 741662819b535072e57b6d0a1d765a326d64febb Mon Sep 17 00:00:00 2001 From: hjpotter92 Date: Thu, 8 Aug 2024 10:02:20 +0530 Subject: [PATCH 7/9] Remove doubling the shasum from embedded message content --- .github/discord-embed-webhook.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/discord-embed-webhook.py b/.github/discord-embed-webhook.py index 58a39dc5d9..63afc0074b 100644 --- a/.github/discord-embed-webhook.py +++ b/.github/discord-embed-webhook.py @@ -21,7 +21,6 @@ def get_embeds(embed: DiscordEmbed, ref_name: str, checksums: list[str]): ) title = filename[9:].split(".")[0] embed.add_embed_field(name=title, value=download_url, inline=False) - embed.add_embed_field(name="SHA256 checksum", value=shasum, inline=False) def main(args): From 71f5980046cbbf9b16c19beb0a92240d840dc826 Mon Sep 17 00:00:00 2001 From: hjpotter92 Date: Thu, 8 Aug 2024 14:17:14 +0530 Subject: [PATCH 8/9] discord-webhook: Fail script when discord webhook doesn't succeed --- .github/discord-embed-webhook.py | 10 ++- upload_build.sh | 136 ------------------------------- 2 files changed, 7 insertions(+), 139 deletions(-) delete mode 100755 upload_build.sh diff --git a/.github/discord-embed-webhook.py b/.github/discord-embed-webhook.py index 63afc0074b..00f093630c 100644 --- a/.github/discord-embed-webhook.py +++ b/.github/discord-embed-webhook.py @@ -3,6 +3,7 @@ import json import pathlib +from requests import Response from discord_webhook import DiscordEmbed, DiscordWebhook @@ -14,11 +15,13 @@ def get_embeds(embed: DiscordEmbed, ref_name: str, checksums: list[str]): for line in checksums: - shasum, filename = line.split() + _, filename = line.split() download_url = DOWNLOAD_BASE_URL_TEMPLATE.format( version=ref_name, filename=filename, ) + # strip the initial `livepeer-` prefix and then identify build + # platform/architecture/release-tag values from file name title = filename[9:].split(".")[0] embed.add_embed_field(name=title, value=download_url, inline=False) @@ -48,8 +51,9 @@ def main(args): get_embeds(embed, args.ref_name, checksums) embed.set_timestamp() webhook.add_embed(embed) - response = webhook.execute() - print(response) + response: Response = webhook.execute() + # Fail the script if discord returns anything except OK status + assert response.ok, "Discord webhook failed" if __name__ == "__main__": diff --git a/upload_build.sh b/upload_build.sh deleted file mode 100755 index 891dbd1042..0000000000 --- a/upload_build.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash - -# CI script for uploading builds. - -set -e -set -o nounset - -BASE_DIR="$(realpath $(dirname "$0"))" - -cd "$BASE_DIR" -RELEASES_DIR="${BASE_DIR}/${RELEASES_DIR:-releases}/" - -mkdir -p "$RELEASES_DIR" - -if [[ "${GOOS:-}" != "" ]]; then - PLATFORM="$GOOS" -elif [[ $(uname) == *"MSYS"* ]]; then - PLATFORM="windows" -else - PLATFORM=$(uname | tr '[:upper:]' '[:lower:]') -fi - -EXT="" -if [[ "$PLATFORM" == "windows" ]]; then - EXT=".exe" -fi -if [[ "$PLATFORM" != "linux" ]] && [[ "$PLATFORM" != "darwin" ]] && [[ "$PLATFORM" != "windows" ]]; then - echo "Unknown/unsupported platform: $PLATFORM" - exit 1 -fi - -if [[ -n "${RELEASE_TAG:-}" ]]; then - PLATFORM="$PLATFORM-$RELEASE_TAG" -fi - -ARCH="$(uname -m)" - -if [[ "${GOARCH:-}" != "" ]]; then - ARCH="$GOARCH" -fi -if [[ "$ARCH" == "x86_64" ]]; then - ARCH="amd64" -fi -if [[ "$ARCH" == "aarch64" ]]; then - ARCH="arm64" -fi -if [[ "$ARCH" != "amd64" ]] && [[ "$ARCH" != "arm64" ]]; then - echo "Unknown/unsupported architecture: $ARCH" - exit 1 -fi - -BASE="livepeer-$PLATFORM-$ARCH" -BRANCH="${TRAVIS_BRANCH:-unknown}" -if [[ "${GHA_REF:-}" != "" ]]; then - BRANCH="$(echo $GHA_REF | sed 's:refs/heads/::')" -fi -VERSION="$(./print_version.sh)" -if echo $VERSION | grep dirty; then - echo "Error: git state dirty, refusing to upload build" - git diff | cat - git status - exit 1 -fi - -# If we want to build with branch --> network support for any other networks, add them here! -NETWORK_BRANCHES="rinkeby mainnet" -# If the binaries are built off a network branch then the resource path should include the network branch name i.e. X.Y.Z/rinkeby or X.Y.Z/mainnet -# If the binaries are not built off a network then the resource path should only include the version i.e. X.Y.Z -VERSION_AND_NETWORK=$VERSION -for networkBranch in $NETWORK_BRANCHES; do - if [[ $BRANCH == "$networkBranch" ]]; then - VERSION_AND_NETWORK="$VERSION/$BRANCH" - fi -done - -NODE="./livepeer${EXT}" -CLI="./livepeer_cli${EXT}" -BENCH="./livepeer_bench${EXT}" -ROUTER="./livepeer_router${EXT}" - -mkdir -p "${BASE_DIR}/$BASE" - -# Optionally step into build directory, if set anywhere -cd "${GO_BUILD_DIR:-./}" -cp "$NODE" "$CLI" "$BENCH" "$ROUTER" "${BASE_DIR}/$BASE" -cd - - -# do a basic upload so we know if stuff's working prior to doing everything else -if [[ $PLATFORM == "windows" ]]; then - FILE="$BASE.zip" - zip -r "${RELEASES_DIR}/$FILE" ./$BASE -else - FILE="$BASE.tar.gz" - tar -czvf "${RELEASES_DIR}/$FILE" ./$BASE -fi - -cd "$RELEASES_DIR" - -FILE_SHA256=$(shasum -a 256 ${FILE}) - -if [[ "${GCLOUD_KEY:-}" == "" ]]; then - echo "GCLOUD_KEY not found, not uploading to Google Cloud." - exit 0 -fi - -# https://stackoverflow.com/a/44751929/990590 -BUCKET="build.livepeer.live" -PROJECT="go-livepeer" -BUCKET_PATH="${PROJECT}/${VERSION_AND_NETWORK}/${FILE}" -resource="/${BUCKET}/${BUCKET_PATH}" -contentType="application/x-compressed-tar" -dateValue="$(date -R)" -stringToSign="PUT\n\n${contentType}\n${dateValue}\n${resource}" -signature="$(echo -en ${stringToSign} | openssl sha1 -hmac ${GCLOUD_SECRET} -binary | base64)" -fullUrl="https://storage.googleapis.com${resource}" - -# Failsafe - don't overwrite existing uploads! -if curl --head --fail $fullUrl 2>/dev/null; then - echo "$fullUrl already exists, not overwriting!" - exit 0 -fi - -curl -X PUT -T "${FILE}" \ - -H "Host: storage.googleapis.com" \ - -H "Date: ${dateValue}" \ - -H "Content-Type: ${contentType}" \ - -H "Authorization: AWS ${GCLOUD_KEY}:${signature}" \ - $fullUrl - -echo "upload done" - -curl -X POST --fail -s \ - -H "Content-Type: application/json" \ - -d "{\"content\": \"Build succeeded ✅\nBranch: $BRANCH\nPlatform: $PLATFORM-$ARCH\nLast commit: $(git log -1 --pretty=format:'%s by %an')\nhttps://build.livepeer.live/${BUCKET_PATH}\nSHA256:\n${FILE_SHA256}\"}" \ - $DISCORD_URL -echo "done" From 1842f1e0be4a3b038be72df4c374c217fba3fe0b Mon Sep 17 00:00:00 2001 From: hjpotter92 Date: Fri, 9 Aug 2024 12:19:47 +0530 Subject: [PATCH 9/9] discord-embed.py: Update script based on PR comments --- .github/discord-embed-webhook.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/discord-embed-webhook.py b/.github/discord-embed-webhook.py index 00f093630c..5671c6eaad 100644 --- a/.github/discord-embed-webhook.py +++ b/.github/discord-embed-webhook.py @@ -20,9 +20,7 @@ def get_embeds(embed: DiscordEmbed, ref_name: str, checksums: list[str]): version=ref_name, filename=filename, ) - # strip the initial `livepeer-` prefix and then identify build - # platform/architecture/release-tag values from file name - title = filename[9:].split(".")[0] + title = filename.lstrip("livepeer-").split(".")[0] embed.add_embed_field(name=title, value=download_url, inline=False) @@ -53,7 +51,9 @@ def main(args): webhook.add_embed(embed) response: Response = webhook.execute() # Fail the script if discord returns anything except OK status - assert response.ok, "Discord webhook failed" + assert ( + response.ok + ), f"Discord webhook failed {response.status_code} {response.content}" if __name__ == "__main__":