diff --git a/.github/actions/bake-test-push/action.yml b/.github/actions/bake-test-push/action.yml index 066e0bc9..10436d91 100644 --- a/.github/actions/bake-test-push/action.yml +++ b/.github/actions/bake-test-push/action.yml @@ -17,6 +17,10 @@ inputs: description: Flag to test image once built default: true type: boolean + scan-image: + description: Flag to scan image for vulnerabilities once built + default: true + type: boolean push-image: description: Flag to push image once built default: false @@ -37,6 +41,14 @@ inputs: description: JSON for authenticating Google Cloud Platform default: "" type: string + snyk-org: + description: Organization ID for Snyk + default: "" + type: string + snyk-token: + description: Token for authenticating with Snyk + default: "" + type: string runs: using: "composite" @@ -47,6 +59,13 @@ runs: env: GITHUB_TOKEN: ${{ inputs.ghcr-token }} + - uses: snyk/actions/setup@master + + - name: Snyk auth + shell: bash + run: | + snyk auth ${{ inputs.snyk-token }} + - uses: actions/setup-python@v5 with: python-version: '3.12' @@ -109,7 +128,30 @@ runs: - name: Test shell: bash run: | - just test "${{ inputs.target }}" "${{ inputs.bakefile }}" + if [[ "${{ inputs.test-image }}" == "true" ]]; then + just test "${{ inputs.target }}" "${{ inputs.bakefile }}" + fi + + - name: Scan + continue-on-error: true + env: + SNYK_ORG: ${{ inputs.snyk-org }} + shell: bash + run: | + if [[ "${{ inputs.scan-image }}" == "true" ]]; then + if [[ "${{ inputs.push-image }}" == "true" ]]; then + just snyk-monitor "${{ inputs.target }}" "${{ inputs.bakefile }}" + else + just snyk-test "${{ inputs.target }}" "${{ inputs.bakefile }}" + fi + fi + + - name: Upload results + uses: github/codeql-action/upload-sarif@v3 + continue-on-error: true + with: + sarif_file: "container.sarif" + category: "${{ inputs.target }}-snyk-vulnerabilities" - name: Push - ${{ inputs.push-image }} uses: docker/bake-action@v4 diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml deleted file mode 100644 index 11198efa..00000000 --- a/.github/actions/build-test-scan-push/action.yaml +++ /dev/null @@ -1,180 +0,0 @@ -name: 'Build/Test/Scan/Push Image' -inputs: - context: - description: Path to the directory of the Dockerfile - required: true - type: string - os: - description: Target OS to build, the same as the extension of the Dockerfile - default: ubuntu2204 - type: string - product: - description: Product being built - type: string - build-args: - description: JSON list of build args for the built image - required: true - type: string - image-tags: - description: List of tags for the built image - required: true - type: string - test-image: - description: Flag to test image once built - default: true - type: boolean - snyk-token: - description: Username for authentication with Snyk for scanning images - type: string - snyk-org-id: - description: Snyk Organization ID to publish scans to - type: string - push-image: - description: Flag to push image once built - default: false - type: boolean - ghcr-token: - description: Username for authentication with GHCR.io - required: true - type: string - dockerhub-username: - description: Username for authentication with DockerHub - required: true - type: string - dockerhub-token: - description: Username for authentication with DockerHub - required: true - type: string - gcp-json: - description: JSON for authenticating Google Cloud Platform - default: "" - type: string - -runs: - using: "composite" - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: false - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: false - - - name: Login to ghcr.io - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ inputs.ghcr-token }} - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ inputs.dockerhub-username }} - password: ${{ inputs.dockerhub-token }} - - - name: Login to GCAR us-central1 - continue-on-error: true - uses: docker/login-action@v3 - with: - registry: us-central1-docker.pkg.dev - username: _json_key - password: '${{ inputs.gcp-json }}' - - - name: Login to GCAR us - continue-on-error: true - uses: docker/login-action@v3 - with: - registry: us-docker.pkg.dev - username: _json_key - password: '${{ inputs.gcp-json }}' - - - name: Login to GCAR asia - continue-on-error: true - uses: docker/login-action@v3 - with: - registry: asia-docker.pkg.dev - username: _json_key - password: '${{ inputs.gcp-json }}' - - - name: Login to GCAR europe - continue-on-error: true - uses: docker/login-action@v3 - with: - registry: europe-docker.pkg.dev - username: _json_key - password: '${{ inputs.gcp-json }}' - - - name: Build - id: image-build - uses: docker/build-push-action@v5 - with: - load: true - context: ${{ inputs.context }} - file: ${{ inputs.context }}/Dockerfile.${{ inputs.os }} - build-args: | - ${{ inputs.build-args }} - tags: ${{ inputs.image-tags }} - - - name: Get first tag - shell: bash - id: first-tag - run: | - IMG_TAGS="${{ inputs.image-tags }}" - FIRST_TAG=$(cut -d "," -f 1 <<< "${IMG_TAGS//$'\n'/}") - echo "$FIRST_TAG" - echo "FIRST_TAG=$FIRST_TAG" >> $GITHUB_OUTPUT - - # We have to use bash logic because step "if"s don't work in composite actions - - name: Test - ${{ inputs.test-image }} - shell: bash - run: | - if [[ "${{ inputs.test-image }}" == "true" ]]; then - echo "${{ inputs.build-args }}" > ${{ inputs.context }}/.env - echo "OS=${{ inputs.os }}" >> ${{ inputs.context }}/.env - cat ${{ inputs.context }}/.env - IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut - fi - - - name: Evaluate Snyk command - id: eval-snyk-command - shell: bash - run: | - if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - SNYK_COMMAND="monitor" - else - SNYK_COMMAND="test" - fi - echo "SNYK_COMMAND=$SNYK_COMMAND" >> $GITHUB_OUTPUT - - - name: Run Snyk ${{ steps.eval-snyk-command.outputs.SNYK_COMMAND }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ inputs.snyk-token }} - with: - image: ${{ steps.first-tag.outputs.FIRST_TAG }} - args: | - --file=${{ inputs.context }}/Dockerfile.${{ inputs.os }} \ - --org=${{ inputs.snyk-org-id }} \ - --project-name=${{ steps.first-tag.FIRST_TAG }} \ - --tags=product=${{ inputs.product }},os=${{ inputs.os }} \ - --exclude-base-image-vulns \ - --app-vulns - command: ${{ steps.eval-snyk-command.outputs.SNYK_COMMAND }} - - - name: Push - ${{ inputs.push-image }} - uses: docker/build-push-action@v5 - with: - push: ${{ inputs.push-image }} - context: ${{ inputs.context }} - file: ${{ inputs.context }}/Dockerfile.${{ inputs.os }} - build-args: ${{ inputs.build-args }} - tags: ${{ inputs.image-tags }} diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml index aedddb10..c95dea69 100644 --- a/.github/workflows/build-bake-preview.yaml +++ b/.github/workflows/build-bake-preview.yaml @@ -99,6 +99,8 @@ jobs: ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' connect-daily: needs: [versions] @@ -137,6 +139,8 @@ jobs: ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' connect-content-init-daily: needs: [versions] @@ -175,6 +179,8 @@ jobs: ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' package-manager-preview: needs: [versions] @@ -213,6 +219,8 @@ jobs: ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' package-manager-daily: needs: [versions] @@ -251,6 +259,8 @@ jobs: ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' r-session-complete-preview: needs: [versions] @@ -289,6 +299,8 @@ jobs: ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' r-session-complete-daily: needs: [versions] @@ -327,6 +339,8 @@ jobs: ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' workbench-preview: needs: [versions] @@ -365,6 +379,8 @@ jobs: ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' workbench-daily: needs: [versions] @@ -403,3 +419,5 @@ jobs: ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index a1c999b2..50b2cd45 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -63,6 +63,8 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' connect: needs: [setup] @@ -96,6 +98,8 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' connect-content-init: needs: [setup] @@ -129,6 +133,8 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' content: needs: [setup] @@ -162,6 +168,8 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' package-manager: needs: [setup] @@ -195,6 +203,8 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' r-session-complete: needs: [setup] @@ -228,6 +238,8 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' workbench: needs: [setup] @@ -261,6 +273,8 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' workbench-for-google-cloud-workstations: needs: [setup] @@ -294,6 +308,8 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' workbench-for-microsoft-azure-ml: needs: [setup] @@ -327,3 +343,5 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + snyk-org: ${{ secrets.SNYK_ORG }} + snyk-token: '${{ secrets.SNYK_TOKEN }}' diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 54f9f6d3..8503bb73 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -33,7 +33,7 @@ jobs: steps: - name: Check Out Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run Hadolint uses: hadolint/hadolint-action@v3.0.0 diff --git a/Justfile b/Justfile index 341c831a..6f264186 100644 --- a/Justfile +++ b/Justfile @@ -24,6 +24,8 @@ PYTHON_VERSION_ALT_RHEL := "3.8.15" QUARTO_VERSION := "1.4.557" +SNYK_ORG := env("SNYK_ORG", "") + export RSC_LICENSE := "" export RSPM_LICENSE := "" export RSW_LICENSE := "" @@ -115,6 +117,65 @@ preview-test target="default" branch="$(git branch --show-current)": BRANCH="${BRANCH}" \ python3 {{justfile_directory()}}/tools/test_bake_artifacts.py --file docker-bake.preview.hcl --target "{{target}}" +# just snyk-code-test +snyk-code-test: + snyk code test --org="{{SNYK_ORG}}" --sarif-file-output=code.sarif {{justfile_directory()}} + +# just snyk-test workbench +snyk-test target="default" file="docker-bake.hcl" *opts="": + SNYK_ORG="{{SNYK_ORG}}" \ + GIT_SHA=$(git rev-parse --short HEAD) \ + python3 {{justfile_directory()}}/tools/snyk_bake_artifacts.py --target "{{target}}" --file "{{file}}" test {{opts}} + +# just snyk-monitor workbench +snyk-monitor target="default" file="docker-bake.hcl" *opts="": + SNYK_ORG="{{SNYK_ORG}}" \ + GIT_SHA=$(git rev-parse --short HEAD) \ + python3 {{justfile_directory()}}/tools/snyk_bake_artifacts.py --target "{{target}}" --file "{{file}}" monitor {{opts}} + +# just snyk-sbom workbench +snyk-sbom target="default" file="docker-bake.hcl" *opts="": + SNYK_ORG="{{SNYK_ORG}}" \ + GIT_SHA=$(git rev-parse --short HEAD) \ + python3 {{justfile_directory()}}/tools/snyk_bake_artifacts.py --target "{{target}}" --file "{{file}}" sbom {{opts}} + +# just snyk-ignore workbench SNYK-XXXX-XXXX-XXXX "Reported upstream in " 2024-08-31 +snyk-ignore context snyk_id reason expiry: + snyk ignore --id="{{snyk_id}}" --reason="{{reason}}" --expiry="{{expiry}}" --policy-path="{{context}}" + +# just preview-snyk-test workbench +preview-snyk-test target="default" branch="$(git branch --show-current)" *opts="": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + SNYK_ORG="{{SNYK_ORG}}" \ + GIT_SHA=$(git rev-parse --short HEAD) \ + python3 {{justfile_directory()}}/tools/snyk_bake_artifacts.py --target "{{target}}" --file "docker-bake.preview.hcl" test {{opts}} + +# just snyk-monitor workbench +preview-snyk-monitor target="default" branch="$(git branch --show-current)" *opts="": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + SNYK_ORG="{{SNYK_ORG}}" \ + GIT_SHA=$(git rev-parse --short HEAD) \ + python3 {{justfile_directory()}}/tools/snyk_bake_artifacts.py --target "{{target}}" --file "docker-bake.preview.hcl" monitor {{opts}} + +# just snyk-sbom workbench +preview-snyk-sbom target="default" branch="$(git branch --show-current)" *opts="": + WORKBENCH_DAILY_VERSION=$(just -f ci.Justfile get-version workbench --type=daily --local) \ + WORKBENCH_PREVIEW_VERSION=$(just -f ci.Justfile get-version workbench --type=preview --local) \ + PACKAGE_MANAGER_DAILY_VERSION=$(just -f ci.Justfile get-version package-manager --type=daily --local) \ + CONNECT_DAILY_VERSION=$(just -f ci.Justfile get-version connect --type=daily --local) \ + BRANCH="{{branch}}" \ + SNYK_ORG="{{SNYK_ORG}}" \ + GIT_SHA=$(git rev-parse --short HEAD) \ + python3 {{justfile_directory()}}/tools/snyk_bake_artifacts.py --target "{{target}}" --file "docker-bake.preview.hcl" sbom {{opts}} + # just lint workbench ubuntu2204 lint $PRODUCT $OS: #!/usr/bin/env bash diff --git a/connect/.snyk b/connect/.snyk new file mode 100644 index 00000000..76241fea --- /dev/null +++ b/connect/.snyk @@ -0,0 +1,10 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-GOLANG-GITHUBCOMJACKCPGXV4-7416900: + - '*': + reason: 'Reported upstream in https://github.com/rstudio/connect/issues/27482' + expires: 2024-07-31T00:00:00.000Z + created: 2024-07-03T13:49:12.040Z +patch: {} diff --git a/connect/Dockerfile.ubuntu2204 b/connect/Dockerfile.ubuntu2204 index 593fe1d3..d4d72986 100644 --- a/connect/Dockerfile.ubuntu2204 +++ b/connect/Dockerfile.ubuntu2204 @@ -12,7 +12,7 @@ ARG QUARTO_VERSION=1.4.557 ARG SCRIPTS_DIR=/opt/positscripts ### Install Quarto ### -RUN QUARTO_VERSION=${QUARTO_VERSION} ${SCRIPTS_DIR}/install_quarto.sh \ +RUN QUARTO_VERSION=${QUARTO_VERSION} ${SCRIPTS_DIR}/install_quarto.sh --install-tinytex --add-path-tinytex \ && ln -s /opt/quarto/${QUARTO_VERSION}/bin/quarto /usr/local/bin/quarto SHELL [ "/bin/bash", "-o", "pipefail", "-c"] diff --git a/connect/test/goss.yaml b/connect/test/goss.yaml index 88d87b2d..63cffa16 100644 --- a/connect/test/goss.yaml +++ b/connect/test/goss.yaml @@ -114,3 +114,10 @@ command: "/usr/local/bin/quarto check --quiet": title: quarto_check exit-status: 0 + +# Ensure TinyTeX is installed + "quarto list tools": + title: quarto_tinytex_installed + exit-status: 0 + stderr: + - "/tinytex\\s+Up to date\\s+v\\d{4}\\.\\d{2}\\.\\d{2}\\s+v\\d{4}\\.\\d{2}\\.\\d{2}/" diff --git a/docker-bake.hcl b/docker-bake.hcl index ac6fe5fa..bf9957c3 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -72,16 +72,16 @@ function get_ubuntu_tags { "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}-${tag_safe_version(product_version)}", "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}-${clean_version(product_version)}", "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}-${clean_version(product_version)}--${GIT_SHA}", - "ghcr.io/rstudio/${product}:${os}", "ghcr.io/rstudio/${product}:${get_os_alt_name(os)}", + "ghcr.io/rstudio/${product}:${os}", "docker.io/rstudio/${product}:${get_os_alt_name(os)}-${tag_safe_version(product_version)}", "docker.io/rstudio/${product}:${get_os_alt_name(os)}-${clean_version(product_version)}", "docker.io/rstudio/${product}:${get_os_alt_name(os)}-${clean_version(product_version)}--${GIT_SHA}", "docker.io/rstudio/${product}:${os}-${tag_safe_version(product_version)}", "docker.io/rstudio/${product}:${os}-${clean_version(product_version)}", "docker.io/rstudio/${product}:${os}-${clean_version(product_version)}--${GIT_SHA}", - "docker.io/rstudio/${product}:${os}", "docker.io/rstudio/${product}:${get_os_alt_name(os)}", + "docker.io/rstudio/${product}:${os}", ] } @@ -184,7 +184,7 @@ variable WORKBENCH_BUILD_MATRIX { variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { default = { builds = [ - {os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + {os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, ] } } @@ -471,6 +471,9 @@ target "workbench-for-google-cloud-workstations" { dockerfile = "Dockerfile.${builds.os}" context = "workbench-for-google-cloud-workstations" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX args = { diff --git a/package-manager/.snyk b/package-manager/.snyk new file mode 100644 index 00000000..e550808b --- /dev/null +++ b/package-manager/.snyk @@ -0,0 +1,12 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-GOLANG-GITHUBCOMJACKCPGXV4-7416900: + - '*': + reason: >- + Reported upstream in + https://github.com/rstudio/package-manager/issues/13981 + expires: 2024-10-01T00:00:00.000Z + created: 2024-07-03T14:03:16.019Z +patch: {} diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204 index 9d1f2d35..a7658da0 100644 --- a/package-manager/Dockerfile.ubuntu2204 +++ b/package-manager/Dockerfile.ubuntu2204 @@ -35,11 +35,12 @@ RUN mkdir -p /var/run/rstudio-pm \ USER rstudio-pm COPY rstudio-pm.gcfg /etc/rstudio-pm/rstudio-pm.gcfg +RUN echo "source <(rspm completion bash)" >> ~/.bashrc \ # Set up licensing to work in userspace mode. This will not prevent activating a # license as root, but it is required to activate one as the non-root user at -# runtime. It's possible for this to fail and the trail will be considered over, +# runtime. It's possible for this to fail and the trial will be considered over, # in which case we can ignore it anyway. -RUN license-manager initialize --userspace || true + && license-manager initialize --userspace || true ENTRYPOINT ["tini", "--"] CMD ["/usr/local/bin/startup.sh"] diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index 649dabbf..c10a69ef 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -30,14 +30,6 @@ RUN gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 595E85A6B1 && chmod +x /tini \ && ln -s /tini /usr/local/bin/tini -### Install TinyTeX ### -SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN curl -sL "https://yihui.org/tinytex/install-bin-unix.sh" | sh \ - && /root/.TinyTeX/bin/*/tlmgr path remove \ - && mv /root/.TinyTeX/ /opt/TinyTeX \ - && /opt/TinyTeX/bin/*/tlmgr option sys_bin /usr/local/bin \ - && /opt/TinyTeX/bin/*/tlmgr path add - ### Install R versions ### RUN R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh \ && R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh \ diff --git a/product/base/scripts/ubuntu/install_quarto.sh b/product/base/scripts/ubuntu/install_quarto.sh index e6071b13..632eebf3 100755 --- a/product/base/scripts/ubuntu/install_quarto.sh +++ b/product/base/scripts/ubuntu/install_quarto.sh @@ -16,18 +16,34 @@ usage() { echo " QUARTO_VERSION=1.3.340 $0" echo "" echo "Options:" - echo " -d, --debug Enable debug output" - echo " -h, --help Print usage and exit" - echo " --prefix Install Quarto to a custom prefix" - echo " Each version of Quarto will have its own subdirectory" - echo " Default: /opt/quarto" + echo " -d, --debug Enable debug output" + echo " -h, --help Print usage and exit" + echo " --prefix Install Quarto to a custom prefix" + echo " Each version of Quarto will have its own subdirectory" + echo " Default: /opt/quarto" + echo " --install-tinytex Install TinyTeX using Quarto" + echo " --add-path-tinytex Add TinyTeX to PATH using Quarto" + echo " --update-tinytex Update TinyTeX using Quarto" + echo " --uninstall-tinytex Uninstall TinyTeX from Quarto" } # Set defaults PREFIX="/opt/quarto" +QUARTO_PATH="${PREFIX}/${QUARTO_VERSION}/bin/quarto" +ADD_PATH_TINYTEX=0 +INSTALL_TINYTEX=0 +UPDATE_TINYTEX=0 +UNINSTALL_TINYTEX=0 +IS_WORKBENCH_INSTALLATION=0 -OPTIONS=$(getopt -o hdr: --long help,debug,prefix: -- "$@") +# Set Quarto Path to the bundled version in Workbench if it exists +if [ -f "/lib/rstudio-server/bin/quarto/bin/quarto" ]; then + QUARTO_PATH="/lib/rstudio-server/bin/quarto/bin/quarto" + IS_WORKBENCH_INSTALLATION=1 +fi + +OPTIONS=$(getopt -o hd --long help,debug,prefix:,install-tinytex,add-path-tinytex,update-tinytex,uninstall-tinytex -- "$@") # shellcheck disable=SC2181 if [[ $? -ne 0 ]]; then exit 1; @@ -49,13 +65,29 @@ while true; do PREFIX="$2" shift 2 ;; + --install-tinytex) + INSTALL_TINYTEX=1 + shift + ;; + --add-path-tinytex) + ADD_PATH_TINYTEX=1 + shift + ;; + --update-tinytex) + UPDATE_TINYTEX=1 + shift + ;; + --uninstall-tinytex) + UNINSTALL_TINYTEX=1 + shift + ;; --) shift; break ;; esac done -if [ -z "$QUARTO_VERSION" ]; then +if [ -z "$QUARTO_VERSION" ] && [[ "$IS_WORKBENCH_INSTALLATION" -eq 0 ]]; then usage exit 1 fi @@ -63,7 +95,7 @@ fi install_quarto() { # Check if Quarto is already installed # shellcheck disable=SC2086 - if ${PREFIX}/${QUARTO_VERSION}/bin/quarto --version | grep -qE "^${QUARTO_VERSION}" ; then + if $QUARTO_PATH --version | grep -qE "^${QUARTO_VERSION}" ; then echo "$d Quarto $QUARTO_VERSION is already installed in $PREFIX/$QUARTO_VERSION $d" return fi @@ -72,4 +104,33 @@ install_quarto() { wget -q -O - "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" | tar xzf - -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 } -install_quarto +update_tinytex() { + $QUARTO_PATH update tinytex --no-prompt +} + +uninstall_tinytex() { + $QUARTO_PATH uninstall tinytex --no-prompt +} + +install_tinytex() { + uninstall_tinytex + if [[ "$ADD_PATH_TINYTEX" -eq 1 ]]; then + $QUARTO_PATH install tinytex --update-path --no-prompt + else + $QUARTO_PATH install tinytex --no-prompt + fi +} + +if [[ "$IS_WORKBENCH_INSTALLATION" -eq 0 ]]; then + # Skip installation if Quarto is bundled with Workbench + install_quarto +fi +if [[ "$INSTALL_TINYTEX" -eq 1 ]]; then + install_tinytex +fi +if [[ "$UPDATE_TINYTEX" -eq 1 ]]; then + update_tinytex +fi +if [[ "$UNINSTALL_TINYTEX" -eq 1 ]]; then + uninstall_tinytex +fi diff --git a/r-session-complete/.snyk b/r-session-complete/.snyk new file mode 100644 index 00000000..095a845b --- /dev/null +++ b/r-session-complete/.snyk @@ -0,0 +1,17 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-GOLANG-GITHUBCOMCREWJAMSAML-5971016: + - '*': + reason: >- + Reported upstream in + https://github.com/rstudio/rstudio-pro/issues/6529 + expires: 2024-08-31T00:00:00.000Z + created: 2024-07-02T20:33:30.847Z + SNYK-GOLANG-GITHUBCOMGOJOSEGOJOSEV3-6070737: + - '*': + reason: 'Reported upstream in https://github.com/rstudio/openid/issues/18' + expires: 2024-07-31T00:00:00.000Z + created: 2024-07-02T20:52:24.627Z +patch: {} diff --git a/r-session-complete/Dockerfile.ubuntu2204 b/r-session-complete/Dockerfile.ubuntu2204 index 8a608cc5..ec3a8658 100644 --- a/r-session-complete/Dockerfile.ubuntu2204 +++ b/r-session-complete/Dockerfile.ubuntu2204 @@ -40,6 +40,9 @@ RUN apt-get update \ ### Install Quarto to PATH ### RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto +### Install TinyTeX using Quarto ### +RUN $SCRIPTS_DIR/install_quarto.sh --install-tinytex --add-path-tinytex + COPY maybe_install_vs_code.sh /tmp/maybe_install_vs_code.sh RUN /tmp/maybe_install_vs_code.sh \ && rm /tmp/maybe_install_vs_code.sh diff --git a/r-session-complete/test/goss.yaml b/r-session-complete/test/goss.yaml index 1c7bf336..44e611ec 100644 --- a/r-session-complete/test/goss.yaml +++ b/r-session-complete/test/goss.yaml @@ -60,3 +60,10 @@ command: "/usr/local/bin/quarto check --quiet": title: quarto_check exit-status: 0 + +# Ensure TinyTeX is installed + "quarto list tools": + title: quarto_tinytex_installed + exit-status: 0 + stderr: + - "/tinytex\\s+Up to date\\s+v\\d{4}\\.\\d{2}\\.\\d{2}\\s+v\\d{4}\\.\\d{2}\\.\\d{2}/" diff --git a/r-session-complete/vscode.extensions.conf b/r-session-complete/vscode.extensions.conf index 11b0a817..0e38faab 100644 --- a/r-session-complete/vscode.extensions.conf +++ b/r-session-complete/vscode.extensions.conf @@ -1,3 +1,4 @@ quarto.quarto -REditorSupport.r@2.6.1 +REditorSupport.r@2.8.2 ms-python.python@2023.6.1 +posit.shiny diff --git a/tools/snyk_bake_artifacts.py b/tools/snyk_bake_artifacts.py new file mode 100644 index 00000000..1983b43e --- /dev/null +++ b/tools/snyk_bake_artifacts.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +""" +Run Snyk vulnerability scanning against bake artifacts by group/target and build definition. + +./snyk_bake_artifacts.py --file --target +""" + +import argparse +import json +import logging +import os +import subprocess +import sys +from pathlib import Path + +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +LOGGER = logging.getLogger(__name__) +SNYK_ORG = os.getenv("SNYK_ORG") +SERVICE_IMAGES = ["workbench-for-microsoft-azure=ml", "workbench-for-google-cloud-workstations"] + +PROJECT_DIR = Path(__file__).resolve().parents[1] + + +parser = argparse.ArgumentParser( + description="Extract a snyk container command from a bake plan" +) +parser.add_argument("--file", default="docker-bake.hcl") +parser.add_argument("--target", default="default") +parser.add_argument("command", choices=["test", "monitor", "sbom"]) +parser.add_argument( + "opts", + nargs="*", + help="Additional options to pass to snyk container command in the format of 'option=value' with no leading '--'. If no options are provided, a default set of options will be used.", +) + + +def get_bake_plan(bake_file="docker-bake.hcl", target="default"): + cmd = ["docker", "buildx", "bake", "-f", str(PROJECT_DIR / bake_file), "--print", target] + run_env = os.environ.copy() + p = subprocess.run(cmd, capture_output=True, env=run_env) + if p.returncode != 0: + LOGGER.error(f"Failed to get bake plan: {p.stderr}") + exit(1) + return json.loads(p.stdout.decode("utf-8")) + + +def render_options(opts): + rendered = [f"--{opt}" for opt in opts] + return rendered + + +def get_version(target_spec): + version = target_spec["args"].get("RSW_VERSION") or target_spec["args"].get("RSC_VERSION") or target_spec["args"].get("RSPM_VERSION") + return version + + +def get_image_type(target_spec): + if target_spec["context"] in SERVICE_IMAGES: + return "service" + return "generic" + + +def build_snyk_command(target_name, target_spec, snyk_command, opts): + context_path = PROJECT_DIR / target_spec["context"] + docker_file_path = context_path / "Dockerfile.ubuntu2204" # TODO: make operating system extension dynamic + cmd = [ + "snyk", + "container", + snyk_command, + ] + if opts: + cmd.extend(render_options(opts)) + else: + if snyk_command == "test": + cmd.extend([ + "--format=legacy", + f"--org={SNYK_ORG}", + f"--file={str(docker_file_path)}", + "--platform=linux/amd64", + f"--project-name={target_spec['tags'][-1]}", + f"--sarif-file-output=container.sarif", + "--severity-threshold=high", + f"--policy-path={target_spec['context']}", + ]) + if "product" not in target_spec["context"]: + cmd.append("--exclude-base-image-vulns") + elif snyk_command == "monitor": + cmd.extend([ + "--format=legacy", + f"--org={SNYK_ORG}", + f"--file={str(docker_file_path)}", + "--platform=linux/amd64", + f"--project-name={target_spec['tags'][-1]}", + "--project-environment=distributed", + "--project-lifecycle=production", + f"--policy-path={target_spec['context']}", + ]) + if "product" not in target_spec["context"]: + cmd.append("--exclude-base-image-vulns") + tags = f"--project-tags=product={target_spec['context']},image_tag={target_spec['tags'][1]},os_distro=ubuntu,os_version=22.04" + version = get_version(target_spec) + if version: + tags += f",version={version}" + image_type = get_image_type(target_spec) + tags += f",image_type={image_type}" + cmd.append(tags) + elif snyk_command == "sbom": + cmd.append("--format=cyclonedx1.4+json") + cmd.append(target_spec["tags"][0]) + if snyk_command == "sbom": + cmd.append(f"> {target_name}_sbom.json") + return cmd + + +def run_cmd(target_name, cmd): + LOGGER.info(f"Running tests for {target_name}") + LOGGER.info(f"{' '.join(cmd)}") + p = subprocess.run(" ".join(cmd), shell=True) + if p.returncode != 0: + LOGGER.error(f"{target_name} test failed with exit code {p.returncode}") + return p.returncode + + +def main(): + args = parser.parse_args() + plan = get_bake_plan(args.file, args.target) + result = 0 + failed_targets = [] + targets = {} + for k in plan["group"][args.target]["targets"]: + for target_name, target_spec in plan["target"].items(): + if target_name.startswith(k): + targets[target_name] = target_spec + LOGGER.info(f"Testing {len(targets.keys())} targets: {targets.keys()}") + for target_name, target_spec in targets.items(): + cmd = build_snyk_command(target_name, target_spec, args.command, args.opts) + LOGGER.debug(" ".join(cmd)) + return_code = run_cmd(target_name, cmd) + if return_code != 0: + failed_targets.append(target_name) + result = 1 + LOGGER.info(f"Failed targets: {failed_targets}") + exit(result) + + +if __name__ == "__main__": + main() diff --git a/workbench-for-google-cloud-workstations/.dockerignore b/workbench-for-google-cloud-workstations/.dockerignore new file mode 100644 index 00000000..1bdfee60 --- /dev/null +++ b/workbench-for-google-cloud-workstations/.dockerignore @@ -0,0 +1,2 @@ +conf/launcher.pem +conf/launcher.pub diff --git a/workbench-for-google-cloud-workstations/.snyk b/workbench-for-google-cloud-workstations/.snyk new file mode 100644 index 00000000..87558195 --- /dev/null +++ b/workbench-for-google-cloud-workstations/.snyk @@ -0,0 +1,22 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-GOLANG-GITHUBCOMCREWJAMSAML-5971016: + - '*': + reason: >- + Reported upstream in + https://github.com/rstudio/rstudio-pro/issues/6529 + expires: 2024-08-31T00:00:00.000Z + created: 2024-07-02T20:33:30.847Z + SNYK-GOLANG-GITHUBCOMGOJOSEGOJOSEV3-6070737: + - '*': + reason: 'Reported upstream in https://github.com/rstudio/openid/issues/18' + expires: 2024-07-31T00:00:00.000Z + created: 2024-07-02T20:52:24.627Z + SNYK-GOLANG-GOLANGORGXNETHTTP2-6531285: + - '*': + reason: 'Patched in later version https://cloud.google.com/support/bulletins#gcp-2024-023' + expires: 2024-07-31T00:00:00.000Z + created: 2024-07-03T16:16:45.000Z +patch: {} diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 index 6347f4f7..324371c9 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 @@ -1,3 +1,4 @@ +FROM product-base-pro as posit_base FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest as build ### ARG declarations ### @@ -28,56 +29,39 @@ ENV DIAGNOSTIC_ONLY false ENV LICENSE_MANAGER_PATH /opt/rstudio-license ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter +### Copy scripts from Posit Base ### +COPY --from=posit_base /opt/positscripts /opt/positscripts + ### Copy package lists and install scripts ### -COPY deps/* / +COPY deps/* /tmp/ ### Update/upgrade system packages ### +COPY deps/apt_packages.txt /tmp/apt_packages.txt RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \ - && apt-get update --fix-missing \ - && apt-get upgrade -yq \ - && xargs -a /apt_packages.txt apt-get install -yq --no-install-recommends \ - && rm /apt_packages.txt \ - && rm -rf /var/lib/apt/lists/* + && ${SCRIPTS_DIR}/apt.sh --update upgrade \ + && ${SCRIPTS_DIR}/apt.sh install $(cat /tmp/apt_packages.txt) \ + && ${SCRIPTS_DIR}/apt.sh --clean \ + && rm /tmp/apt_packages.txt ### Install R versions ### -RUN curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \ - && apt-get update \ - && apt-get install -yq --no-install-recommends ./r-${R_VERSION}_1_amd64.deb \ - && apt-get install -yq --no-install-recommends ./r-${R_VERSION_ALT}_1_amd64.deb \ - && rm -f ./r-${R_VERSION}_1_amd64.deb \ - && rm -f ./r-${R_VERSION_ALT}_1_amd64.deb \ - && ln -s /opt/R/${R_VERSION}/bin/R /usr/local/bin/R \ - && ln -s /opt/R/${R_VERSION}/bin/Rscript /usr/local/bin/Rscript \ - && rm -rf /var/lib/apt/lists/* +COPY deps/r_packages.txt /tmp/r_packages.txt +RUN ${SCRIPTS_DIR}/apt.sh --update \ + && R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh -r /tmp/r_packages.txt \ + && R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh -r /tmp/r_packages.txt \ + && ${SCRIPTS_DIR}/apt.sh --clean \ + && ln -s /opt/R/${R_VERSION} /opt/R/default \ + && ln -s /opt/R/default/bin/R /usr/local/bin/R \ + && ln -s /opt/R/default/bin/Rscript /usr/local/bin/Rscript \ + && rm -f /tmp/r_packages.txt ### Install Python versions ### -RUN curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \ - && apt-get update \ - && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION}_1_amd64.deb \ - && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION_ALT}_1_amd64.deb \ - && rm -rf python-${PYTHON_VERSION}_1_amd64.deb \ - && rm -rf python-${PYTHON_VERSION_ALT}_1_amd64.deb \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m ensurepip --upgrade \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade pip \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m ensurepip --upgrade \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade pip \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \ - && rm -rf /var/lib/apt/lists/* - -### Install basic data science packages for Python and R ### -RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install -r /py_packages.txt \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install -r /py_packages.txt \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \ - && ./install_r_packages.sh \ - && rm install_r_packages.sh py_packages.txt r_packages.txt +COPY deps/requirements.txt /tmp/requirements.txt +RUN ${SCRIPTS_DIR}/apt.sh --update \ + && PYTHON_VERSION=${PYTHON_VERSION} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ + && PYTHON_VERSION=${PYTHON_VERSION_ALT} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ + && ${SCRIPTS_DIR}/apt.sh --clean \ + && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default \ + && rm -f /tmp/requirements.txt ### Locale configuration ### RUN localedef -i en_US -f UTF-8 en_US.UTF-8 @@ -85,17 +69,11 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -### Install Quarto to PATH ### -RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto - ### Install Pro Drivers ### -RUN apt-get update \ - && apt-get install -yq --no-install-recommends unixodbc unixodbc-dev \ - && curl -O https://cdn.rstudio.com/drivers/7C152C12/installer/rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ - && apt-get update \ - && apt-get install -yq --no-install-recommends ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ - && rm -f ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ - && rm -rf /var/lib/apt/lists/* \ +RUN ${SCRIPTS_DIR}/apt.sh --update upgrade \ + && ${SCRIPTS_DIR}/apt.sh install unixodbc unixodbc-dev \ + && DRIVERS_VERSION=${DRIVERS_VERSION} ${SCRIPTS_DIR}/install_drivers.sh \ + && ${SCRIPTS_DIR}/apt.sh --clean \ && cp /opt/rstudio-drivers/odbcinst.ini.sample /etc/odbcinst.ini \ && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/jammy/latest")' @@ -109,17 +87,30 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION && dpkg-sig --verify ./rstudio-workbench.deb \ && apt-get update \ && apt-get install -y --no-install-recommends ./rstudio-workbench.deb \ + # a wild hack to ensure that workbench can install _and start_ completely before shutdown + && sleep 30 \ && rm ./rstudio-workbench.deb \ + && apt-get remove -yq dpkg-sig \ && apt-get autoremove -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/rstudio-server/r-versions +### Install Quarto to PATH ### +RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto + +### Install TinyTeX using Quarto ### +RUN $SCRIPTS_DIR/install_quarto.sh --install-tinytex --add-path-tinytex + +# Workaround to ensure no pre-generated certificates are included in image distributions. +# This happens in the step immediately following Workbench installation in case the certificates are generated. +RUN rm -f /etc/rstudio/launcher.pem /etc/rstudio/launcher.pub + ### Install GCW License Manager ### # TODO(ianpittwood): Replace monitor download with $RSW_VERSION after upgrading to 2023.06.0 RUN mkdir -p /opt/rstudio-license/ \ && mkdir -p /var/lib/rstudio-workbench \ - && curl -sL "https://s3.amazonaws.com/rstudio-ide-build/monitor/jammy/rsp-monitor-workbench-gcpw-amd64-2023.06.0-419.pro1.tar.gz" | \ + && curl -sL "https://s3.amazonaws.com/rstudio-ide-build/monitor/jammy/rsp-monitor-workbench-gcpw-amd64-${RSW_VERSION//+/-}.tar.gz" | \ tar xzvf - --strip 2 -C /opt/rstudio-license/ \ && chmod 0755 /opt/rstudio-license/license-manager \ && mv /opt/rstudio-license/license-manager /opt/rstudio-license/license-manager-orig \ @@ -161,18 +152,10 @@ COPY startup/* /startup/base/ COPY supervisord.conf /etc/supervisor/supervisord.conf COPY --chmod=600 sssd.conf /etc/sssd/sssd.conf COPY conf/* /etc/rstudio/ -COPY --chmod=600 conf/launcher.p* /etc/rstudio # GCW specific COPY --chmod=755 workstation-startup/* /etc/workstation-startup.d/ COPY --chmod=644 jupyter/jupyter_notebook_config.json /opt/python/jupyter/etc/jupyter/jupyter_notebook_config.json -### Clean up ### -RUN apt-get remove -yq dpkg-sig \ - && apt-get install -yqf --no-install-recommends \ - && apt-get autoremove -yq \ - && apt-get clean -yq \ - && rm -rf /var/lib/apt/lists/* - EXPOSE 80/tcp EXPOSE 5559/tcp diff --git a/workbench-for-google-cloud-workstations/conf/launcher-env b/workbench-for-google-cloud-workstations/conf/launcher-env index 40f05ea8..d8127a52 100644 --- a/workbench-for-google-cloud-workstations/conf/launcher-env +++ b/workbench-for-google-cloud-workstations/conf/launcher-env @@ -3,4 +3,4 @@ Environment: LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 JobType: any -Environment: PATH=/opt/python/3.10.14/bin:$PATH +Environment: PATH=/opt/python/default/bin:$PATH diff --git a/workbench-for-google-cloud-workstations/deps/apt_packages.txt b/workbench-for-google-cloud-workstations/deps/apt_packages.txt index bf6d9c10..df9f9911 100644 --- a/workbench-for-google-cloud-workstations/deps/apt_packages.txt +++ b/workbench-for-google-cloud-workstations/deps/apt_packages.txt @@ -22,6 +22,7 @@ libuser1-dev libxext6 libxrender1 locales +lsb-release oddjob-mkhomedir rrdtool sssd diff --git a/workbench-for-google-cloud-workstations/deps/install_r_packages.sh b/workbench-for-google-cloud-workstations/deps/install_r_packages.sh deleted file mode 100755 index 885ee27f..00000000 --- a/workbench-for-google-cloud-workstations/deps/install_r_packages.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -ex - -UBUNTU_CODENAME=$(lsb_release -cs) -CRAN_REPO="https://packagemanager.posit.co/cran/__linux__/${UBUNTU_CODENAME}/latest" - -r_packages=$(awk '{print "\"" $0 "\""}' r_packages.txt | paste -d',' -s -) -/opt/R/${R_VERSION}/bin/R --slave -e "install.packages(c(${r_packages}), repos = \"${CRAN_REPO}\")" -/opt/R/${R_VERSION_ALT}/bin/R --slave -e "install.packages(c(${r_packages}), repos = \"${CRAN_REPO}\")" diff --git a/workbench-for-google-cloud-workstations/deps/py_packages.txt b/workbench-for-google-cloud-workstations/deps/requirements.txt similarity index 100% rename from workbench-for-google-cloud-workstations/deps/py_packages.txt rename to workbench-for-google-cloud-workstations/deps/requirements.txt diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index c55ff6d8..547a503a 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -204,7 +204,7 @@ command: ] {{ $python_version := .Env.PYTHON_VERSION }} {{ $python_version_alt := .Env.PYTHON_VERSION_ALT }} - {{ $py_package_list := readFile "/tmp/deps/py_packages.txt" | splitList "\n" }} + {{ $py_package_list := readFile "/tmp/deps/requirements.txt" | splitList "\n" }} {{- range $py_package_list }} Check Python {{ $python_version }} has "{{.}}" installed: exec: /opt/python/{{$python_version}}/bin/pip show {{.}} @@ -233,3 +233,10 @@ command: "/usr/local/bin/quarto check --quiet": title: quarto_check exit-status: 0 + +# Ensure TinyTeX is installed + "quarto list tools": + title: quarto_tinytex_installed + exit-status: 0 + stderr: + - "/tinytex\\s+Up to date\\s+v\\d{4}\\.\\d{2}\\.\\d{2}\\s+v\\d{4}\\.\\d{2}\\.\\d{2}/" diff --git a/workbench-for-microsoft-azure-ml/.dockerignore b/workbench-for-microsoft-azure-ml/.dockerignore new file mode 100644 index 00000000..1bdfee60 --- /dev/null +++ b/workbench-for-microsoft-azure-ml/.dockerignore @@ -0,0 +1,2 @@ +conf/launcher.pem +conf/launcher.pub diff --git a/workbench-for-microsoft-azure-ml/.gitignore b/workbench-for-microsoft-azure-ml/.gitignore new file mode 100644 index 00000000..1bdfee60 --- /dev/null +++ b/workbench-for-microsoft-azure-ml/.gitignore @@ -0,0 +1,2 @@ +conf/launcher.pem +conf/launcher.pub diff --git a/workbench-for-microsoft-azure-ml/.snyk b/workbench-for-microsoft-azure-ml/.snyk new file mode 100644 index 00000000..095a845b --- /dev/null +++ b/workbench-for-microsoft-azure-ml/.snyk @@ -0,0 +1,17 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-GOLANG-GITHUBCOMCREWJAMSAML-5971016: + - '*': + reason: >- + Reported upstream in + https://github.com/rstudio/rstudio-pro/issues/6529 + expires: 2024-08-31T00:00:00.000Z + created: 2024-07-02T20:33:30.847Z + SNYK-GOLANG-GITHUBCOMGOJOSEGOJOSEV3-6070737: + - '*': + reason: 'Reported upstream in https://github.com/rstudio/openid/issues/18' + expires: 2024-07-31T00:00:00.000Z + created: 2024-07-02T20:52:24.627Z +patch: {} diff --git a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 index c77416b9..bcb6139d 100644 --- a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 +++ b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 @@ -46,6 +46,8 @@ RUN apt-get update \ && gpg --keyserver keys.openpgp.org --recv-keys 51C0B5BB19F92D60 \ && dpkg-sig --verify ./rstudio-workbench.deb \ && apt-get install -yq --no-install-recommends ./rstudio-workbench.deb \ + # a wild hack to ensure that workbench can install _and start_ completely before shutdown + && sleep 30 \ && rm ./rstudio-workbench.deb \ && mkdir -p /opt/rstudio-license/ \ && mkdir -p /var/lib/rstudio-workbench/ \ @@ -59,9 +61,16 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/rstudio-server/r-versions +# Workaround to ensure no pre-generated certificates are included in image distributions. +# This happens in the step immediately following Workbench installation in case the certificates are generated. +RUN rm -f /etc/rstudio/launcher.pem /etc/rstudio/launcher.pub + ### Install Quarto to PATH ### RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto +### Install TinyTeX using Quarto ### +RUN $SCRIPTS_DIR/install_quarto.sh --install-tinytex --add-path-tinytex + COPY --chmod=0755 license-manager-shim /opt/rstudio-license/license-manager COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh COPY startup/* /startup/base/ diff --git a/workbench-for-microsoft-azure-ml/conf/launcher-env b/workbench-for-microsoft-azure-ml/conf/launcher-env index 4ae3a261..d8127a52 100644 --- a/workbench-for-microsoft-azure-ml/conf/launcher-env +++ b/workbench-for-microsoft-azure-ml/conf/launcher-env @@ -3,4 +3,4 @@ Environment: LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 JobType: any -Environment: PATH=/opt/python/3.6.5/bin:$PATH +Environment: PATH=/opt/python/default/bin:$PATH diff --git a/workbench-for-microsoft-azure-ml/test/goss.yaml b/workbench-for-microsoft-azure-ml/test/goss.yaml index b94fae54..332ad325 100644 --- a/workbench-for-microsoft-azure-ml/test/goss.yaml +++ b/workbench-for-microsoft-azure-ml/test/goss.yaml @@ -151,3 +151,10 @@ command: "/usr/local/bin/quarto check --quiet": title: quarto_check exit-status: 0 + +# Ensure TinyTeX is installed + "quarto list tools": + title: quarto_tinytex_installed + exit-status: 0 + stderr: + - "/tinytex\\s+Up to date\\s+v\\d{4}\\.\\d{2}\\.\\d{2}\\s+v\\d{4}\\.\\d{2}\\.\\d{2}/" diff --git a/workbench/.dockerignore b/workbench/.dockerignore new file mode 100644 index 00000000..1bdfee60 --- /dev/null +++ b/workbench/.dockerignore @@ -0,0 +1,2 @@ +conf/launcher.pem +conf/launcher.pub diff --git a/workbench/.snyk b/workbench/.snyk new file mode 100644 index 00000000..095a845b --- /dev/null +++ b/workbench/.snyk @@ -0,0 +1,17 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-GOLANG-GITHUBCOMCREWJAMSAML-5971016: + - '*': + reason: >- + Reported upstream in + https://github.com/rstudio/rstudio-pro/issues/6529 + expires: 2024-08-31T00:00:00.000Z + created: 2024-07-02T20:33:30.847Z + SNYK-GOLANG-GITHUBCOMGOJOSEGOJOSEV3-6070737: + - '*': + reason: 'Reported upstream in https://github.com/rstudio/openid/issues/18' + expires: 2024-07-31T00:00:00.000Z + created: 2024-07-02T20:52:24.627Z +patch: {} diff --git a/workbench/Dockerfile.ubuntu2204 b/workbench/Dockerfile.ubuntu2204 index 712dab62..c6d0cb1f 100644 --- a/workbench/Dockerfile.ubuntu2204 +++ b/workbench/Dockerfile.ubuntu2204 @@ -63,9 +63,16 @@ RUN apt-get update \ && rm -rf /var/lib/rstudio-server/r-versions \ && rm -rf /var/lib/rstudio-launcher/Local/jobs/buildkitsandbox +# Workaround to ensure no pre-generated certificates are included in image distributions. +# This happens in the step immediately following Workbench installation in case the certificates are generated. +RUN rm -f /etc/rstudio/launcher.pem /etc/rstudio/launcher.pub + ### Install Quarto to PATH ### RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto +### Install TinyTeX using Quarto ### +RUN $SCRIPTS_DIR/install_quarto.sh --install-tinytex --add-path-tinytex + COPY maybe_install_vs_code.sh /tmp/maybe_install_vs_code.sh RUN /tmp/maybe_install_vs_code.sh \ && rm /tmp/maybe_install_vs_code.sh diff --git a/workbench/conf/launcher-env b/workbench/conf/launcher-env index 4ae3a261..d8127a52 100644 --- a/workbench/conf/launcher-env +++ b/workbench/conf/launcher-env @@ -3,4 +3,4 @@ Environment: LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 JobType: any -Environment: PATH=/opt/python/3.6.5/bin:$PATH +Environment: PATH=/opt/python/default/bin:$PATH diff --git a/workbench/test/goss.yaml b/workbench/test/goss.yaml index bfbee8d0..a7a2beb2 100644 --- a/workbench/test/goss.yaml +++ b/workbench/test/goss.yaml @@ -148,3 +148,10 @@ command: "/usr/local/bin/quarto check --quiet": title: quarto_check exit-status: 0 + +# Ensure TinyTeX is installed + "quarto list tools": + title: quarto_tinytex_installed + exit-status: 0 + stderr: + - "/tinytex\\s+Up to date\\s+v\\d{4}\\.\\d{2}\\.\\d{2}\\s+v\\d{4}\\.\\d{2}\\.\\d{2}/"