fix(make): add target adhocpkgs (#27) #26
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CID | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - beta | |
| paths-ignore: ["docs/**", "scripts/**", "**.md", "*", "!pyproject.toml", "!flake.nix", "!*.lock"] | |
| pull_request: | |
| types: [opened, synchronize, labeled, reopened, ready_for_review] | |
| paths-ignore: ["docs/**", "scripts/**", "**.md", "*", "!pyproject.toml", "!flake.nix", "!*.lock"] | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| description: "Run with tmate.io debugging enabled" | |
| required: true | |
| type: boolean | |
| default: false | |
| run_build_images: | |
| description: "Run build-images job" | |
| required: false | |
| type: boolean | |
| default: false | |
| run_execute_workflow: | |
| description: "Run execute-workflow job" | |
| required: false | |
| type: boolean | |
| default: false | |
| workflow_execution_mode: | |
| description: "Workflow execution mode" | |
| required: false | |
| type: string | |
| default: "prod" | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| actions: write | |
| contents: read | |
| jobs: | |
| set-variables: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| debug: ${{ steps.set-variables.outputs.debug }} | |
| skip_ci: ${{ steps.set-variables.outputs.skip_ci }} | |
| skip_tests: ${{ steps.set-variables.outputs.skip_tests }} | |
| mode: ${{ steps.set-variables.outputs.mode }} | |
| steps: | |
| - name: Set action variables | |
| id: set-variables | |
| run: | | |
| DEBUG="false" | |
| MODE="prod" | |
| SKIP_CI="false" | |
| SKIP_TESTS="false" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| DEBUG="${{ inputs.debug_enabled }}" | |
| MODE="${{ inputs.workflow_execution_mode }}" | |
| fi | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'skip-ci') }}; then | |
| SKIP_CI="true" | |
| fi | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'skip-tests') }}; then | |
| SKIP_TESTS="true" | |
| fi | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'actions-debug') }}; then | |
| DEBUG="true" | |
| fi | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'workflow-mode-dev') }}; then | |
| MODE="dev" | |
| fi | |
| fi | |
| echo "DEBUG=$DEBUG" | |
| echo "MODE=$MODE" | |
| echo "SKIP_CI=$SKIP_CI" | |
| echo "SKIP_TESTS=$SKIP_TESTS" | |
| echo "DEBUG=$DEBUG" >> $GITHUB_OUTPUT | |
| echo "MODE=$MODE" >> $GITHUB_OUTPUT | |
| echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT | |
| echo "SKIP_TESTS=$SKIP_TESTS" >> $GITHUB_OUTPUT | |
| config-workflows: | |
| needs: [set-variables] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| config-path: ${{ steps.config-output.outputs.path }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} | |
| - name: Create Flyte config from YAML template | |
| id: yq-process | |
| uses: mikefarah/yq@master | |
| with: | |
| cmd: "yq e \ | |
| '.admin.endpoint = strenv(FLYTE_CLUSTER_ENDPOINT) | \ | |
| .storage.stow.config.project_id = strenv(GCP_PROJECT_ID) | \ | |
| .storage.stow.config.scopes = strenv(GCP_STORAGE_SCOPES) | \ | |
| .storage.container = strenv(GCP_STORAGE_CONTAINER)' \ | |
| .flyte/config-template.yaml > .flyte/config.yaml" | |
| env: | |
| FLYTE_CLUSTER_ENDPOINT: ${{ secrets.FLYTE_CLUSTER_ENDPOINT }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| GCP_STORAGE_SCOPES: ${{ secrets.GCP_STORAGE_SCOPES }} | |
| GCP_STORAGE_CONTAINER: ${{ secrets.GCP_STORAGE_CONTAINER }} | |
| - name: Upload Flyte config as an artifact | |
| id: config-output | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: flyte-config | |
| path: ${{ secrets.FLYTECTL_CONFIG }} | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: [set-variables, config-workflows] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }} | |
| concurrency: | |
| group: test-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }} | |
| cancel-in-progress: true | |
| strategy: | |
| matrix: | |
| python_version: ['3.10'] | |
| steps: | |
| - name: Check Variables | |
| run: | | |
| echo "SKIP_CI=${{ needs.set-variables.outputs.skip_ci }}" | |
| echo "SKIP_TESTS=${{ needs.set-variables.outputs.skip_tests }}" | |
| echo "DEBUG=${{ needs.set-variables.outputs.debug }}" | |
| echo "MODE=${{ needs.set-variables.outputs.mode }}" | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} | |
| - name: Setup environment | |
| uses: ./.github/actions/setup_environment | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| debug_enabled: ${{ needs.set-variables.outputs.debug }} | |
| - name: Download Flyte config | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: flyte-config | |
| path: .flyte/ | |
| - name: "Setup tmate debug session" | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ inputs.debug_enabled }} | |
| - name: Lint and typecheck | |
| run: | | |
| make lint-check | |
| - name: Run tests | |
| run: | | |
| make test-cov-xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: false | |
| verbose: true | |
| test-nix: | |
| runs-on: ubuntu-latest | |
| needs: [set-variables, config-workflows] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }} | |
| concurrency: | |
| group: test-nix-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} | |
| - uses: DeterminateSystems/nix-installer-action@v9 | |
| - uses: DeterminateSystems/magic-nix-cache-action@v2 | |
| - run: nix flake check --impure --accept-flake-config | |
| - run: nix build . --accept-flake-config | |
| build-docker-images: | |
| needs: [set-variables] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'build-images') || contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_build_images)) }} | |
| uses: ./.github/workflows/build-images.yaml | |
| concurrency: | |
| group: bi-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }} | |
| cancel-in-progress: true | |
| secrets: | |
| GCP_ARTIFACT_REGISTRY_PATH: ${{ secrets.GCP_ARTIFACT_REGISTRY_PATH }} | |
| with: | |
| debug_enabled: ${{ needs.set-variables.outputs.debug }} | |
| mode: ${{ needs.set-variables.outputs.mode }} | |
| build-nix-images: | |
| runs-on: ubuntu-latest | |
| needs: [set-variables] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && ( contains(github.event.pull_request.labels.*.name, 'build-images') || contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_build_images) || (github.event_name == 'workflow_dispatch' && inputs.run_execute_workflow) ) }} | |
| concurrency: | |
| group: bni-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| packages: write | |
| environment: | |
| name: release | |
| url: https://ghcr.io/sciexp/flytezendev | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} | |
| - uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: arm64 | |
| - uses: DeterminateSystems/nix-installer-action@v9 | |
| with: | |
| extra-conf: | | |
| extra-platforms = aarch64-linux | |
| - uses: DeterminateSystems/magic-nix-cache-action@v2 | |
| - uses: rlespinasse/github-slug-action@v4 | |
| with: | |
| prefix: CI_ | |
| - name: Set git variables | |
| run: | | |
| echo "GIT_REPO_NAME=$CI_GITHUB_REPOSITORY_NAME_PART" >> $GITHUB_ENV | |
| echo "GIT_REF=$CI_GITHUB_REF_NAME" >> $GITHUB_ENV | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| echo "GIT_SHA=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA" >> $GITHUB_ENV | |
| echo "GIT_SHA_SHORT=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT" >> $GITHUB_ENV | |
| else | |
| echo "GIT_SHA=$CI_GITHUB_SHA" >> $GITHUB_ENV | |
| echo "GIT_SHA_SHORT=$CI_GITHUB_SHA_SHORT" >> $GITHUB_ENV | |
| fi | |
| - name: Set nix variables | |
| run: | | |
| echo "NIX_IMAGE_SYSTEMS=x86_64-linux" >> $GITHUB_ENV | |
| - name: Build nix images | |
| run: | | |
| echo "Using Git Repository Name: $GIT_REPO_NAME" | |
| echo "Using Git Reference: $GIT_REF" | |
| echo "Using Git SHA: $GIT_SHA" | |
| echo "Using Git SHA Short: $GIT_SHA_SHORT" | |
| nix run .#devcontainerManifest --impure --accept-flake-config | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| execute-workflow: | |
| needs: [config-workflows, build-nix-images, set-variables] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_execute_workflow)) }} | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: ef-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }} | |
| cancel-in-progress: true | |
| strategy: | |
| matrix: | |
| python_version: ['3.10'] | |
| env: | |
| FLYTECTL_CONFIG: ${{ secrets.FLYTECTL_CONFIG }} | |
| FLYTE_OAUTH_CLIENT_SECRET: ${{ secrets.FLYTE_OAUTH_CLIENT_SECRET }} | |
| WORKFLOW_IMAGE: ${{ vars.WORKFLOW_IMAGE }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} | |
| - name: Setup environment | |
| uses: ./.github/actions/setup_environment | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| debug_enabled: ${{ needs.set-variables.outputs.debug }} | |
| - name: Download Flyte config | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: flyte-config | |
| path: .flyte/ | |
| - name: Setup tmate debug session | |
| if: ${{ inputs.debug_enabled == 'true' }} | |
| uses: mxschmitt/action-tmate@v3 | |
| - name: Execute workflow | |
| id: execute | |
| run: | | |
| make run_${{ needs.set-variables.outputs.mode }} | |
| - name: Create config tarball | |
| id: save-hydra-outputs | |
| run: | | |
| TAR_FILENAME="hydra_outputs_${GITHUB_SHA_SHORT}.tar.gz" | |
| tar -czf $TAR_FILENAME ./outputs/ | |
| tar -tzf $TAR_FILENAME | |
| echo "HYDRA_OUTPUTS_TAR=$TAR_FILENAME" >> $GITHUB_ENV | |
| - name: Upload config artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: hydra-outputs | |
| path: ${{ env.HYDRA_OUTPUTS_TAR }} | |
| release: | |
| runs-on: ubuntu-latest | |
| needs: [test, test-nix] | |
| if: ${{ github.repository_owner == 'sciexp' && github.event_name == 'push' }} | |
| environment: | |
| name: release | |
| url: https://github.com/sciexp/flytezen/releases/tag/${{ steps.semanticrelease.outputs.git-tag }} | |
| permissions: | |
| contents: write | |
| outputs: | |
| version: ${{ steps.semanticrelease.outputs.version }} | |
| released: ${{ steps.semanticrelease.outputs.released }} | |
| git-head: ${{ steps.semanticrelease.outputs.git-head }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} | |
| - uses: DeterminateSystems/nix-installer-action@v9 | |
| - uses: DeterminateSystems/magic-nix-cache-action@v2 | |
| - run: nix profile install .#releaseEnv | |
| - uses: cihelper/action-semanticrelease-poetry@v1 | |
| id: semanticrelease | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ steps.semanticrelease.outputs.released == 'true' }} | |
| with: | |
| name: poetry-build | |
| path: ./dist | |
| publish-release-images: | |
| runs-on: ubuntu-latest | |
| needs: [set-variables, release] | |
| if: ${{ needs.release.outputs.released == 'true' }} | |
| concurrency: | |
| group: bri-${{ github.workflow }}-${{ github.ref }}-${{ github.sha }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| packages: write | |
| environment: | |
| name: release | |
| url: https://ghcr.io/sciexp/flytezendev | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.release.outputs.git-head }} | |
| - uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: arm64 | |
| - uses: DeterminateSystems/nix-installer-action@v9 | |
| with: | |
| extra-conf: | | |
| extra-platforms = aarch64-linux | |
| - uses: DeterminateSystems/magic-nix-cache-action@v2 | |
| - uses: rlespinasse/github-slug-action@v4 | |
| with: | |
| prefix: CI_ | |
| - name: Set git variables | |
| run: | | |
| echo "GIT_REPO_NAME=$CI_GITHUB_REPOSITORY_NAME_PART" >> $GITHUB_ENV | |
| echo "GIT_REF=$CI_GITHUB_REF_NAME" >> $GITHUB_ENV | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| echo "GIT_SHA=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA" >> $GITHUB_ENV | |
| echo "GIT_SHA_SHORT=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT" >> $GITHUB_ENV | |
| else | |
| echo "GIT_SHA=$CI_GITHUB_SHA" >> $GITHUB_ENV | |
| echo "GIT_SHA_SHORT=$CI_GITHUB_SHA_SHORT" >> $GITHUB_ENV | |
| fi | |
| - name: Build nix images | |
| run: | | |
| echo "Using Git Repository Name: $GIT_REPO_NAME" | |
| echo "Using Git Reference: $GIT_REF" | |
| echo "Using Git SHA: $GIT_SHA" | |
| echo "Using Git SHA Short: $GIT_SHA_SHORT" | |
| nix run .#devcontainerManifest --impure --accept-flake-config | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| VERSION: ${{ needs.release.outputs.version }} | |
| publish-pypi: | |
| runs-on: ubuntu-latest | |
| needs: release | |
| if: ${{ needs.release.outputs.released == 'true' }} | |
| permissions: | |
| id-token: write | |
| environment: | |
| name: release | |
| url: https://pypi.org/project/flytezen/${{needs.release.outputs.version}}/ | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: poetry-build | |
| path: ./dist | |
| - uses: pypa/gh-action-pypi-publish@release/v1 | |
| test-docs-build: | |
| runs-on: ubuntu-latest | |
| needs: set-variables | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }} | |
| strategy: | |
| matrix: | |
| python_version: ['3.10'] | |
| permissions: | |
| contents: read | |
| pages: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} | |
| - name: Setup environment | |
| uses: ./.github/actions/setup_environment | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| debug_enabled: ${{ needs.set-variables.outputs.debug }} | |
| - name: Build | |
| run: make docs-build | |
| build-release-docs: | |
| runs-on: ubuntu-latest | |
| needs: [set-variables, release] | |
| if: ${{ needs.release.outputs.released == 'true' }} | |
| strategy: | |
| matrix: | |
| python_version: ['3.10'] | |
| permissions: | |
| contents: read | |
| pages: read | |
| environment: github-pages | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.release.outputs.git-head }} | |
| - name: Setup environment | |
| uses: ./.github/actions/setup_environment | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| debug_enabled: ${{ needs.set-variables.outputs.debug }} | |
| - name: Build | |
| run: make docs-build | |
| - name: Upload artifact | |
| uses: actions/upload-pages-artifact@v2 | |
| with: | |
| path: ./site | |
| publish-docs: | |
| runs-on: ubuntu-latest | |
| needs: [build-release-docs, release] | |
| if: ${{ needs.release.outputs.released == 'true' }} | |
| permissions: | |
| id-token: write | |
| pages: write | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v2 |