From 7e872d55fb38c122caf42c0ffecfb203110f4233 Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:55:39 -0400 Subject: [PATCH 01/11] Add plugin --- build.gradle.kts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 11f19cc45d0..05e700823ec 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -22,8 +22,15 @@ plugins { // ensure the correct version of KGP ends up on our buildscript classpath alias(libs.plugins.kotlin.multiplatform) apply false alias(libs.plugins.kotlin.jvm) apply false + alias(libs.plugins.aws.kotlin.repo.tools.artifactsizemetrics) } +artifactSizeMetrics { + artifactPrefixes = setOf(":services", ":aws-runtime") + closurePrefixes = setOf(":services") + significantChangeThresholdPercentage = 5.0 + projectRepositoryName = "aws-sdk-kotlin" + val testJavaVersion = typedProp("test.java.version")?.let { JavaLanguageVersion.of(it) }?.also { From e4b703dcceb79f3b80ac2f3e8a1eca253590c2e7 Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:56:33 -0400 Subject: [PATCH 02/11] Plugin alias --- gradle/libs.versions.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0c7e57bcf8b..f8b76dd2cf1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,7 +2,7 @@ kotlin-version = "1.9.23" dokka-version = "1.9.10" -aws-kotlin-repo-tools-version = "0.4.0" +aws-kotlin-repo-tools-version = "0.4.4" # libs coroutines-version = "1.7.3" @@ -135,3 +135,4 @@ kotlinx-binary-compatibility-validator = { id = "org.jetbrains.kotlinx.binary-co kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin-version"} aws-kotlin-repo-tools-kmp = { id = "aws.sdk.kotlin.gradle.kmp", version.ref = "aws-kotlin-repo-tools-version" } aws-kotlin-repo-tools-smithybuild = { id = "aws.sdk.kotlin.gradle.smithybuild", version.ref = "aws-kotlin-repo-tools-version" } +aws-kotlin-repo-tools-artifactsizemetrics = { id = "aws.sdk.kotlin.gradle.artifactsizemetrics", version.ref = "aws-kotlin-repo-tools-version" } From 996bca2e16e876c8d260399b4b7054378a1621d7 Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:57:12 -0400 Subject: [PATCH 03/11] Update codebuild-ci.yml --- .github/workflows/codebuild-ci.yml | 152 +++++++++++++++++++++++++++-- 1 file changed, 146 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codebuild-ci.yml b/.github/workflows/codebuild-ci.yml index 201399c6568..48a5b88634a 100644 --- a/.github/workflows/codebuild-ci.yml +++ b/.github/workflows/codebuild-ci.yml @@ -1,9 +1,11 @@ name: AWS CodeBuild CI on: + release: + types: [ published ] pull_request: - branches: - - main + types: [ opened, synchronize, reopened, labeled, unlabeled ] + branches: [ main ] workflow_dispatch: inputs: aws-sdk-kotlin-pr: @@ -27,6 +29,7 @@ env: permissions: id-token: write contents: read + pull-requests: write # Allow one instance of this workflow per pull request, and cancel older runs when new changes are pushed concurrency: @@ -35,6 +38,7 @@ concurrency: jobs: e2e-tests: + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - name: Verify PRs are not running malicious code @@ -63,7 +67,8 @@ jobs: aws codebuild stop-build --id $BUILD_ID fi - service-check-batch: + service-check-batch-and-artifact-size-metrics: + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - name: Verify PRs are not running malicious code @@ -78,14 +83,143 @@ jobs: with: role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} aws-region: us-west-2 - - name: Run Service Check Batch + - name: Run Service Check Batch and Calculate Artifact Size Metrics + id: svc-check-batch + run: | + REPOSITORY=$(echo ${{ github.repository }} | cut -d '/' -f 2) + PULL_REQUEST="" + INTERNAL=${{ github.event.number }} + EXTERNAL=${{ inputs.aws-sdk-kotlin-pr }} + + if [ ! -z "$INTERNAL" ]; then + PULL_REQUEST=${{ github.event.number }} + echo "Identified this an internal pull request build: $PULL_REQUEST" + elif [ ! -z "$EXTERNAL" ]; then + PULL_REQUEST=${{ inputs.aws-sdk-kotlin-pr }} + echo "Identified this as an external pull request build: $PULL_REQUEST" + else + echo "Couldn't identify this as an internal or external pull request" + exit 1 + fi + + .github/scripts/run-codebuild-batch-job.sh \ + --project gh-aws-sdk-kotlin-svc-check-batch \ + --source ${{ github.event.pull_request.head.sha }} \ + --pr-number $PULL_REQUEST \ + --repository $REPOSITORY \ + --external-contributor-sdk-pr-number ${{ inputs.aws-sdk-kotlin-pr }} \ + --external-contributor-smithy-pr-number ${{ inputs.smithy-kotlin-pr }} + - name: Cancel build + if: ${{ cancelled() }} + env: + BUILD_ID: ${{ steps.svc-check-batch.outputs.aws-build-id }} + run: | + if [ ! -z "$BUILD_ID" ]; then + echo "cancelling in-progress batch build: id=$BUILD_ID" + aws codebuild stop-build --id $BUILD_ID + fi + - name: Collect Artifact Size Metrics + run: | + PULL_REQUEST="" + INTERNAL=${{ github.event.number }} + EXTERNAL=${{ inputs.aws-sdk-kotlin-pr }} + + if [ ! -z "$INTERNAL" ]; then + PULL_REQUEST=${{ github.event.number }} + echo "Identified this an internal pull request build: $PULL_REQUEST" + elif [ ! -z "$EXTERNAL" ]; then + PULL_REQUEST=${{ inputs.aws-sdk-kotlin-pr }} + echo "Identified this as an external pull request build: $PULL_REQUEST" + else + echo "Couldn't identify this as an internal or external pull request" + exit 1 + fi + + ./gradlew collectDelegatedArtifactSizeMetrics -PpullRequest=$PULL_REQUEST + - name: Analyze Artifact Size Metrics + run: ./gradlew analyzeArtifactSizeMetrics + - name: Show Results + uses: actions/github-script@v7 + with: + script: | + const getComments = + `query { + repository(owner:"${context.repo.owner}", name:"${context.repo.repo}"){ + pullRequest(number: ${context.issue.number}) { + id + comments(last:100) { + nodes { + id + body + author { + login + } + isMinimized + } + } + } + } + }` + + const response = await github.graphql(getComments) + const comments = response.repository.pullRequest.comments.nodes + + const mutations = comments + .filter(comment => comment.author.login == 'github-actions' && !comment.isMinimized && comment.body.startsWith('Affected Artifacts')) + .map(comment => + github.graphql( + `mutation { + minimizeComment(input:{subjectId:"${comment.id}", classifier:OUTDATED}){ + clientMutationId + } + }` + ) + ) + await Promise.all(mutations) + + const fs = require('node:fs') + const comment = fs.readFileSync('build/reports/metrics/artifact-analysis.md', 'utf8') + + const writeComment = + `mutation { + addComment(input:{body:"""${comment}""", subjectId:"${response.repository.pullRequest.id}"}){ + clientMutationId + } + }` + + await github.graphql(writeComment) + + - name: Evaluate Result + if: ${{ !contains(github.event.pull_request.labels.*.name, 'acknowledge-artifact-size-increase') }} + run: | + cd build/reports/metrics + cat has-significant-change.txt | grep false || { + echo An artifact increased in size by more than allowed or a new artifact was created. + echo If this is expected please add the 'acknowledge-artifact-size-increase' label to this pull request. + exit 1 + } + + release-artifact-size-metrics: + if: github.event_name == 'release' + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v2 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + - name: Calculate Artifact Size Metrics id: svc-check-batch run: | + REPOSITORY=$(echo ${{ github.repository }} | cut -d '/' -f 2) + .github/scripts/run-codebuild-batch-job.sh \ --project gh-aws-sdk-kotlin-svc-check-batch \ --source ${{ github.event.pull_request.head.sha }} \ - --sdk-pr ${{ inputs.aws-sdk-kotlin-pr }} \ - --smithy-pr ${{ inputs.smithy-kotlin-pr }} + --repository $REPOSITORY \ + --release ${{ github.event.release.tag_name }} - name: Cancel build if: ${{ cancelled() }} env: @@ -95,3 +229,9 @@ jobs: echo "cancelling in-progress batch build: id=$BUILD_ID" aws codebuild stop-build --id $BUILD_ID fi + - name: Collect Artifact Size Metrics + run: ./gradlew collectDelegatedArtifactSizeMetrics -Prelease=${{ github.event.release.tag_name }} + - name: Save Artifact Size Metrics + run: ./gradlew saveArtifactSizeMetrics + - name: Put Artifact Size Metrics in CloudWatch + run: ./gradlew putArtifactSizeMetricsInCloudWatch -Prelease=${{ github.event.release.tag_name }} From f305a3dbbeea54747e4b42542ea2be4a14980a5d Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:57:43 -0400 Subject: [PATCH 04/11] Update run-codebuild-batch-job.sh --- .github/scripts/run-codebuild-batch-job.sh | 46 ++++++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/.github/scripts/run-codebuild-batch-job.sh b/.github/scripts/run-codebuild-batch-job.sh index 357427a6a4a..46d585d96b4 100755 --- a/.github/scripts/run-codebuild-batch-job.sh +++ b/.github/scripts/run-codebuild-batch-job.sh @@ -8,8 +8,11 @@ error_exit() { PROJECT_NAME=gh-aws-sdk-kotlin-svc-check-batch # get the source version to be built (defaults to main branch if not specified) SOURCE_VERSION=main -SDK_PR="" -SMITHY_PR="" +GITHUB_RELEASE="" +GITHUB_PULL_REQUEST_NUMBER="" +GITHUB_REPOSITORY_NO_ORG="" +EXTERNAL_CONTRIBUTOR_SDK_PR="" +EXTERNAL_CONTRIBUTOR_SMITHY_PR="" while [[ $# -gt 0 ]]; do case "$1" in @@ -29,19 +32,43 @@ while [[ $# -gt 0 ]]; do shift 2 fi ;; - --sdk-pr) + --release) if [[ "$2" == --* ]]; then shift 1 else - SDK_PR="$2" + GITHUB_RELEASE="$2" shift 2 fi ;; - --smithy-pr) + --pr-number) + if [[ "$2" == --* ]]; then + shift 1 + else + GITHUB_PULL_REQUEST_NUMBER="$2" + shift 2 + fi + ;; + --repository) + if [[ "$2" == --* ]]; then + shift 1 + else + GITHUB_REPOSITORY_NO_ORG="$2" + shift 2 + fi + ;; + --external-contributor-sdk-pr-number) + if [[ "$2" == --* ]]; then + shift 1 + else + EXTERNAL_CONTRIBUTOR_SDK_PR="$2" + shift 2 + fi + ;; + --external-contributor-smithy-pr-number) if [[ "$2" == --* || -z "$2" ]]; then shift 1 else - SMITHY_PR="$2" + EXTERNAL_CONTRIBUTOR_SMITHY_PR="$2" shift 2 fi ;; @@ -52,8 +79,11 @@ while [[ $# -gt 0 ]]; do esac done -export SDK_PR -export SMITHY_PR +export GITHUB_RELEASE +export GITHUB_PULL_REQUEST_NUMBER +export GITHUB_REPOSITORY_NO_ORG +export EXTERNAL_CONTRIBUTOR_SDK_PR +export EXTERNAL_CONTRIBUTOR_SMITHY_PR echo "Starting CodeBuild project ${PROJECT_NAME}" From 89eeb99ecab3b68d93568cc268b02d1cd3cfdfc8 Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Mon, 29 Apr 2024 10:10:47 -0400 Subject: [PATCH 05/11] Add missing bracket --- build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle.kts b/build.gradle.kts index 05e700823ec..f2f6e5accb5 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -30,6 +30,7 @@ artifactSizeMetrics { closurePrefixes = setOf(":services") significantChangeThresholdPercentage = 5.0 projectRepositoryName = "aws-sdk-kotlin" +} val testJavaVersion = typedProp("test.java.version")?.let { JavaLanguageVersion.of(it) From bd2713cedb69f8be2fcb8591c8788ec640091e47 Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Wed, 1 May 2024 12:19:09 -0400 Subject: [PATCH 06/11] Update codebuild-ci.yml --- .github/workflows/codebuild-ci.yml | 89 +----------------------------- 1 file changed, 2 insertions(+), 87 deletions(-) diff --git a/.github/workflows/codebuild-ci.yml b/.github/workflows/codebuild-ci.yml index 48a5b88634a..2385a00e0d2 100644 --- a/.github/workflows/codebuild-ci.yml +++ b/.github/workflows/codebuild-ci.yml @@ -37,38 +37,8 @@ concurrency: cancel-in-progress: true jobs: - e2e-tests: - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - name: Verify PRs are not running malicious code - if: ${{ (inputs.aws-sdk-kotlin-pr != '' || inputs.smithy-kotlin-pr != '') && inputs.check-pr == false }} - run: | - echo Please verify the PRs are not running any malicious code and mark the checkbox true when running the workflow - exit 1 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} - aws-region: us-west-2 - - name: Run E2E Tests - id: e2e-tests - uses: aws-actions/aws-codebuild-run-build@v1 - with: - project-name: gh-aws-sdk-kotlin-e2e-tests - env-vars-for-codebuild: SDK_PR, SMITHY_PR - - name: Cancel build - if: ${{ cancelled() }} - env: - BUILD_ID: ${{ steps.e2e-tests.outputs.aws-build-id }} - run: | - if [ ! -z "$BUILD_ID"]; then - echo "cancelling in-progress build: id=$BUILD_ID" - aws codebuild stop-build --id $BUILD_ID - fi - service-check-batch-and-artifact-size-metrics: - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - name: Verify PRs are not running malicious code @@ -83,61 +53,6 @@ jobs: with: role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} aws-region: us-west-2 - - name: Run Service Check Batch and Calculate Artifact Size Metrics - id: svc-check-batch - run: | - REPOSITORY=$(echo ${{ github.repository }} | cut -d '/' -f 2) - PULL_REQUEST="" - INTERNAL=${{ github.event.number }} - EXTERNAL=${{ inputs.aws-sdk-kotlin-pr }} - - if [ ! -z "$INTERNAL" ]; then - PULL_REQUEST=${{ github.event.number }} - echo "Identified this an internal pull request build: $PULL_REQUEST" - elif [ ! -z "$EXTERNAL" ]; then - PULL_REQUEST=${{ inputs.aws-sdk-kotlin-pr }} - echo "Identified this as an external pull request build: $PULL_REQUEST" - else - echo "Couldn't identify this as an internal or external pull request" - exit 1 - fi - - .github/scripts/run-codebuild-batch-job.sh \ - --project gh-aws-sdk-kotlin-svc-check-batch \ - --source ${{ github.event.pull_request.head.sha }} \ - --pr-number $PULL_REQUEST \ - --repository $REPOSITORY \ - --external-contributor-sdk-pr-number ${{ inputs.aws-sdk-kotlin-pr }} \ - --external-contributor-smithy-pr-number ${{ inputs.smithy-kotlin-pr }} - - name: Cancel build - if: ${{ cancelled() }} - env: - BUILD_ID: ${{ steps.svc-check-batch.outputs.aws-build-id }} - run: | - if [ ! -z "$BUILD_ID" ]; then - echo "cancelling in-progress batch build: id=$BUILD_ID" - aws codebuild stop-build --id $BUILD_ID - fi - - name: Collect Artifact Size Metrics - run: | - PULL_REQUEST="" - INTERNAL=${{ github.event.number }} - EXTERNAL=${{ inputs.aws-sdk-kotlin-pr }} - - if [ ! -z "$INTERNAL" ]; then - PULL_REQUEST=${{ github.event.number }} - echo "Identified this an internal pull request build: $PULL_REQUEST" - elif [ ! -z "$EXTERNAL" ]; then - PULL_REQUEST=${{ inputs.aws-sdk-kotlin-pr }} - echo "Identified this as an external pull request build: $PULL_REQUEST" - else - echo "Couldn't identify this as an internal or external pull request" - exit 1 - fi - - ./gradlew collectDelegatedArtifactSizeMetrics -PpullRequest=$PULL_REQUEST - - name: Analyze Artifact Size Metrics - run: ./gradlew analyzeArtifactSizeMetrics - name: Show Results uses: actions/github-script@v7 with: @@ -182,7 +97,7 @@ jobs: const writeComment = `mutation { - addComment(input:{body:"""${comment}""", subjectId:"${response.repository.pullRequest.id}"}){ + addComment(input:{body:"test", subjectId:"${response.repository.pullRequest.id}"}){ clientMutationId } }` From 5b7866df881a9160aafc9566f675c535824f4572 Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Wed, 1 May 2024 12:26:07 -0400 Subject: [PATCH 07/11] Update codebuild-ci.yml --- .github/workflows/codebuild-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codebuild-ci.yml b/.github/workflows/codebuild-ci.yml index 2385a00e0d2..e7d8a28a39a 100644 --- a/.github/workflows/codebuild-ci.yml +++ b/.github/workflows/codebuild-ci.yml @@ -60,7 +60,7 @@ jobs: const getComments = `query { repository(owner:"${context.repo.owner}", name:"${context.repo.repo}"){ - pullRequest(number: ${context.issue.number}) { + pullRequest(number: ${context.issue.number ?? inputs.aws-sdk-kotlin-pr}) { id comments(last:100) { nodes { From 5438f1a31357bf3c85aaf82c645d856c7e9935ce Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Wed, 1 May 2024 12:33:31 -0400 Subject: [PATCH 08/11] Update codebuild-ci.yml --- .github/workflows/codebuild-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codebuild-ci.yml b/.github/workflows/codebuild-ci.yml index e7d8a28a39a..eb9fbb416f4 100644 --- a/.github/workflows/codebuild-ci.yml +++ b/.github/workflows/codebuild-ci.yml @@ -60,7 +60,7 @@ jobs: const getComments = `query { repository(owner:"${context.repo.owner}", name:"${context.repo.repo}"){ - pullRequest(number: ${context.issue.number ?? inputs.aws-sdk-kotlin-pr}) { + pullRequest(number: ${context.issue.number ?? process.env.SDK_PR}) { id comments(last:100) { nodes { From 78a5f70a41fc9d9afe128ec5a0aaba01b903ddf0 Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Wed, 1 May 2024 12:34:44 -0400 Subject: [PATCH 09/11] Update codebuild-ci.yml --- .github/workflows/codebuild-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/codebuild-ci.yml b/.github/workflows/codebuild-ci.yml index eb9fbb416f4..5b1d782deab 100644 --- a/.github/workflows/codebuild-ci.yml +++ b/.github/workflows/codebuild-ci.yml @@ -92,9 +92,6 @@ jobs: ) await Promise.all(mutations) - const fs = require('node:fs') - const comment = fs.readFileSync('build/reports/metrics/artifact-analysis.md', 'utf8') - const writeComment = `mutation { addComment(input:{body:"test", subjectId:"${response.repository.pullRequest.id}"}){ From 56dd85c05085828839eb7497db346d102424b223 Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Wed, 1 May 2024 12:37:46 -0400 Subject: [PATCH 10/11] Update codebuild-ci.yml --- .github/workflows/codebuild-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codebuild-ci.yml b/.github/workflows/codebuild-ci.yml index 5b1d782deab..a7b83e5c77c 100644 --- a/.github/workflows/codebuild-ci.yml +++ b/.github/workflows/codebuild-ci.yml @@ -38,7 +38,7 @@ concurrency: jobs: service-check-batch-and-artifact-size-metrics: - if: github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - name: Verify PRs are not running malicious code From 91c5edf1bacf9ee844c7d5bbd902e724df2358bd Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Wed, 1 May 2024 12:38:03 -0400 Subject: [PATCH 11/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 413bbdefa7b..de386017507 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# AWS SDK for Kotlin +# AWS SDK for Kotlin! [![Apache 2 licensed][apache-badge]][apache-url] [![Maven][maven-badge]][maven-url] [apache-badge]: https://img.shields.io/badge/license-APACHE2-blue.svg