workflows: Update build workflow to not rely on curl based upload #3117
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| import argparse | ||
| import os | ||
| import json | ||
| import pathlib | ||
|
|
||
| from requests import Response | ||
| from discord_webhook import DiscordEmbed, DiscordWebhook | ||
|
|
||
|
|
||
| GITHUB_CONTEXT_JSON = os.environ.get("GITHUB_CONTEXT_JSON", "{}") | ||
| DOWNLOAD_BASE_URL_TEMPLATE = ( | ||
| "https://build.livepeer.live/go-livepeer/{version}/{filename}" | ||
| ) | ||
|
|
||
|
|
||
| def get_embeds(embed: DiscordEmbed, ref_name: str, checksums: list[str]): | ||
| for line in checksums: | ||
| _, filename = line.split() | ||
| download_url = DOWNLOAD_BASE_URL_TEMPLATE.format( | ||
| version=ref_name, | ||
| filename=filename, | ||
| ) | ||
| title = filename.lstrip("livepeer-").split(".")[0] | ||
| embed.add_embed_field(name=title, value=download_url, inline=False) | ||
|
|
||
|
|
||
| def main(args): | ||
| checksums = [] | ||
| github_context = json.loads(GITHUB_CONTEXT_JSON) | ||
| head = github_context.get("head", {}) | ||
| repo = head.get("repo", {}) | ||
| checksums_file = pathlib.Path("releases") / f"{args.ref_name}_checksums.txt" | ||
| commit_url = f'{repo.get("html_url")}/commit/{head.get("sha")}' | ||
| checksums = checksums_file.read_text().splitlines() | ||
| webhook = DiscordWebhook( | ||
| url=args.discord_url, | ||
| content=":white_check_mark: Build succeeded for go-livepeer :white_check_mark:", | ||
| username="[BOT] Livepeer builder", | ||
| ) | ||
| webhook.add_file(filename=checksums_file.name, file=checksums_file.read_bytes()) | ||
| embed = DiscordEmbed( | ||
| title=head.get("ref"), | ||
| description=args.git_commit, | ||
| color=2928914, | ||
| url=commit_url, | ||
| ) | ||
| embed.add_embed_field(name="Commit SHA", value=head.get("sha")) | ||
| embed.set_author(name=args.git_committer) | ||
| get_embeds(embed, args.ref_name, checksums) | ||
| embed.set_timestamp() | ||
| webhook.add_embed(embed) | ||
| response: Response = webhook.execute() | ||
| # Fail the script if discord returns anything except OK status | ||
| assert ( | ||
| response.ok | ||
| ), f"Discord webhook failed {response.status_code} {response.content}" | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| parser = argparse.ArgumentParser("Discord embed content generator for build system") | ||
| parser.add_argument("--discord-url", help="Discord webhook URL") | ||
| parser.add_argument("--ref-name", help="Tag/branch/commit for current build") | ||
| parser.add_argument("--git-commit", help="git commit message") | ||
| parser.add_argument("--git-committer", help="git commit author name") | ||
| args = parser.parse_args() | ||
| main(args) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,7 +15,7 @@ concurrency: | |
| jobs: | ||
| linux-build: | ||
| name: Build binaries for ${{ matrix.target.GOOS }}-${{ matrix.target.type }}-${{ matrix.target.GOARCH }} | ||
| runs-on: ubuntu-20.04 | ||
| container: | ||
| image: ${{ matrix.target.container }} | ||
| env: | ||
|
|
@@ -26,31 +26,26 @@ jobs: | |
| target: | ||
| - GOOS: linux | ||
| GOARCH: amd64 | ||
| container: ubuntu:20.04 | ||
| type: cpu | ||
|
|
||
| - GOOS: linux | ||
| GOARCH: arm64 | ||
| container: ubuntu:20.04 | ||
| type: cpu | ||
|
|
||
| - GOOS: linux | ||
| GOARCH: amd64 | ||
| container: livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04 | ||
| type: gpu | ||
|
|
||
| - GOOS: linux | ||
| GOARCH: arm64 | ||
| container: livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04 | ||
| type: gpu | ||
|
|
||
| - GOOS: windows | ||
| GOARCH: amd64 | ||
| container: ubuntu:22.04 | ||
| type: cpu | ||
|
|
||
|
|
@@ -78,19 +73,14 @@ jobs: | |
| cache: true | ||
| cache-dependency-path: go.sum | ||
|
|
||
| - name: Set build environment | ||
| run: | | ||
| echo "GOARCH=${{ matrix.target.GOARCH }}" >> $GITHUB_ENV | ||
| echo "GOOS=${{ matrix.target.GOOS }}" >> $GITHUB_ENV | ||
| echo "GO_BUILD_DIR=lp-builds/" >> $GITHUB_ENV | ||
| echo "PKG_CONFIG_PATH=/github/home/compiled/lib/pkgconfig" >> $GITHUB_ENV | ||
| echo "LP_BUILD_DIR=livepeer-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}" >> $GITHUB_ENV | ||
| mkdir -p lp-builds/ releases/ | ||
|
|
||
| - name: Set GPU build environment | ||
| if: matrix.target.type == 'gpu' | ||
|
|
@@ -99,6 +89,7 @@ jobs: | |
| echo "LIBRARY_PATH=/usr/local/cuda_${{ matrix.target.GOARCH }}/lib64" >> $GITHUB_ENV | ||
| echo "CGO_LDFLAGS=-L/usr/local/cuda_${{ matrix.target.GOARCH }}/lib64" >> $GITHUB_ENV | ||
| echo "RELEASE_TAG=gpu" >> $GITHUB_ENV | ||
| echo "LP_BUILD_DIR=livepeer-${{ matrix.target.GOOS }}-gpu-${{ matrix.target.GOARCH }}" >> $GITHUB_ENV | ||
|
|
||
| - name: Install dependencies | ||
| run: | | ||
|
|
@@ -120,7 +111,6 @@ jobs: | |
| run: go mod download | ||
|
|
||
| - name: Install ffmpeg | ||
|
There was a problem hiding this comment. curious why this is being removed since ffmpeg takes a long time to build also seems unrelated to the rest of the PR There was a problem hiding this comment. the cache has not been working forever (screenshot below)
i tried fixing it in There was a problem hiding this comment. Thanks for the context - I am not too familiar with the CI build actions myself. The only thing I can think of is there is a v4 update to the action
weird, maybe it implicitly installs protoc somehow ... but I am not seeing anything obvious in the install script (and my own box doesn't even have it) |
||
| run: ./install_ffmpeg.sh | ||
|
|
||
| - name: Build binaries | ||
|
|
@@ -130,24 +120,28 @@ jobs: | |
| git config --global --add safe.directory '*' | ||
| ./ci_env.sh make | ||
|
|
||
| - name: Archive binaries for windows | ||
| if: matrix.target.GOOS == 'windows' | ||
| run: | | ||
| mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/" | ||
| cd "$GO_BUILD_DIR/" | ||
| find . -type f -exec mv '{}' "$LP_BUILD_DIR/" \; | ||
| zip -9rq "../releases/$LP_BUILD_DIR.zip" ./ | ||
|
|
||
| - name: Archive binaries | ||
| if: matrix.target.GOOS != 'windows' | ||
| run: | | ||
| mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/" | ||
| cd "$GO_BUILD_DIR/" | ||
| find . -type f -exec mv '{}' "$LP_BUILD_DIR" \; | ||
| tar -czvf "../releases/$LP_BUILD_DIR.tar.gz" ./ | ||
|
|
||
| - name: Upload artifacts for cutting release | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: release-artifacts-${{ matrix.target.GOOS }}-${{ matrix.target.type }}-${{ matrix.target.GOARCH }} | ||
| path: releases/ | ||
|
|
||
| macos-build: | ||
| name: Build binaries for ${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }} | ||
| runs-on: ${{ matrix.target.runner }} | ||
|
|
@@ -161,7 +155,7 @@ jobs: | |
|
|
||
| - GOOS: darwin | ||
| GOARCH: arm64 | ||
| runner: macos-14-xlarge | ||
|
|
||
| steps: | ||
| - name: Check out code | ||
|
|
@@ -185,6 +179,8 @@ jobs: | |
| echo "GOARCH=${{ matrix.target.GOARCH }}" >> $GITHUB_ENV | ||
| echo "GOOS=${{ matrix.target.GOOS }}" >> $GITHUB_ENV | ||
| echo "GO_BUILD_DIR=lp-builds/" >> $GITHUB_ENV | ||
| echo "LP_BUILD_DIR=livepeer-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}" >> $GITHUB_ENV | ||
| mkdir -p lp-builds/ releases/ | ||
|
|
||
| - name: Cache ffmpeg | ||
| id: cache-ffmpeg | ||
|
|
@@ -230,24 +226,20 @@ jobs: | |
| app-notarization-team-id: ${{ secrets.CI_MACOS_NOTARIZATION_TEAM_ID }} | ||
| binary-path: "lp-builds/" | ||
|
|
||
| - name: Archive binaries | ||
| if: matrix.platform.name != 'windows' | ||
| run: | | ||
| mkdir -p "${GO_BUILD_DIR}/${LP_BUILD_DIR}/" | ||
| cd "$GO_BUILD_DIR/" | ||
| find . -type f -exec mv '{}' "$LP_BUILD_DIR" \; | ||
| tar -czvf "../releases/${LP_BUILD_DIR}.tar.gz" . | ||
|
|
||
| - name: Upload artifacts for cutting release | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: release-artifacts-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }} | ||
| path: releases/ | ||
|
|
||
| upload: | ||
| name: Upload artifacts to google bucket | ||
| permissions: | ||
|
|
@@ -258,11 +250,20 @@ jobs: | |
| - macos-build | ||
| - linux-build | ||
| steps: | ||
| - name: Check out code | ||
| uses: actions/[email protected] | ||
| with: | ||
| fetch-depth: 0 | ||
| # Check https://github.com/livepeer/go-livepeer/pull/1891 | ||
| # for ref value discussion | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
|
|
||
| - name: Download artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| pattern: release-artifacts-* | ||
| path: releases/ | ||
| merge-multiple: true | ||
|
|
||
| - name: Generate sha256 checksum and gpg signatures for release artifacts | ||
| uses: livepeer/action-gh-checksum-and-gpg-sign@latest | ||
|
|
@@ -289,16 +290,33 @@ jobs: | |
|
|
||
| - name: Upload release archives to Google Cloud | ||
| id: upload-archives | ||
| uses: google-github-actions/upload-cloud-storage@v2 | ||
| with: | ||
| path: "releases" | ||
| destination: "build.livepeer.live/${{ github.event.repository.name }}/${{ (github.ref_type == 'tag' && github.ref_name) || github.sha }}" | ||
| parent: false | ||
| process_gcloudignore: false | ||
|
|
||
| - name: Upload branch manifest file | ||
| id: upload-manifest | ||
| uses: google-github-actions/upload-cloud-storage@v2 | ||
| with: | ||
| path: ${{ steps.branch-manifest.outputs.manifest-file }} | ||
| destination: "build.livepeer.live/${{ github.event.repository.name }}/" | ||
| parent: false | ||
| process_gcloudignore: false | ||
|
|
||
| - name: Trigger discord webhook | ||
|
There was a problem hiding this comment. Which Discord channel receives those messages? A comment in code would be nice |
||
| shell: bash | ||
| env: | ||
| GITHUB_CONTEXT_JSON: ${{ toJson(github.event.pull_request) }} | ||
| run: | | ||
| pip install -U discord-webhook --no-cache-dir | ||
| python .github/discord-embed-webhook.py \ | ||
| --ref-name="${{ (github.ref_type == 'tag' && github.ref_name) || github.sha }}" \ | ||
| --discord-url="${{ secrets.DISCORD_URL }}" \ | ||
| --git-commit="$(git log -1 --pretty=format:'%s')" \ | ||
| --git-committer="$(git log -1 --pretty=format:'%an')" | ||
|
|
||
| - name: Notify new build upload | ||
| run: curl -X POST https://holy-bread-207a.livepeer.workers.dev | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is sth new? Unrelated to the upload issue?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
the upload issue is because gcp deprecated its signature mechanism we were relying on in
upload_build.shscript. it was also sending out separate build messages for each build to the discord channel instead of grouping them together.