wip arm support

.github/workflows/build.yml

@@ -0,0 +1,137 @@
+name: Python builds
+
+on:
+  push:
+    paths:
+      - 'builder/**'
+      - 'test/**'
+      - 'Makefile'
+      - '.github/workflows/build.yml'
+  pull_request:
+    paths:
+      - 'builder/**'
+      - 'test/**'
+      - 'Makefile'
+      - '.github/workflows/build.yml'
+  workflow_dispatch:
+    inputs:
+      platforms:
+        description: |
+          Comma-separated list of platforms. Specify "all" to use all platforms (the default).
+        required: false
+        default: 'all'
+        type: string
+      python_versions:
+        description: |
+          Comma-separated list of Python versions. Specify "last-N" to use the
+          last N minor Python versions, or "all" to use all minor Python versions since Python 3.1.
+          Defaults to "last-5,devel".
+        required: false
+        default: 'last-5,devel'
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  setup-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      platforms: ${{ steps.setup-matrix.outputs.platforms }}
+      python_versions: ${{ steps.setup-matrix.outputs.python_versions }}
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up matrix of platforms and R versions
+      id: setup-matrix
+      run: |
+        platforms=$(python test/get_platforms.py ${{ github.event.inputs.platforms }})
+        echo "platforms=$platforms" >> $GITHUB_OUTPUT
+        python_versions=$(python test/get_python_versions.py ${{ github.event.inputs.python_versions }})
+        echo "python_versions=$python_versions" >> $GITHUB_OUTPUT
+
+  docker-images:
+    needs: setup-matrix
+    strategy:
+      matrix:
+        platform: ${{ fromJson(needs.setup-matrix.outputs.platforms) }}
+    runs-on: ubuntu-latest
+    name: Docker image (${{ matrix.platform }})
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+
+      # Enable Docker layer caching without having to push to a registry.
+      # https://docs.docker.com/build/ci/github-actions/examples/#local-cache
+      # This may eventually be migrated to the GitHub Actions cache backend,
+      # which is still considered experimental.
+      # https://github.com/moby/buildkit#github-actions-cache-experimental
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ matrix.platform }}-buildx-${{ github.sha }}
+          restore-keys: ${{ matrix.platform }}-buildx-
+
+      # Use docker buildx instead of docker-compose here because cache exporting
+      # does not seem to work as of docker-compose v2.6.0 and buildx v0.8.2, even
+      # though it works with buildx individually.
+      - name: Build image
+        run: |
+          docker buildx build -t python-builds:${{ matrix.platform }} \
+            --file builder/Dockerfile.${{ matrix.platform }} \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --cache-to "type=local,dest=/tmp/.buildx-cache-new,mode=max" \
+            builder
+
+      # Temporary workaround for unbounded GHA cache growth with the local cache mode.
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  test:
+    needs: [setup-matrix, docker-images]
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{ fromJson(needs.setup-matrix.outputs.platforms) }}
+        r_version: ${{ fromJson(needs.setup-matrix.outputs.python_versions) }}
+    runs-on: ubuntu-latest
+    name: ${{ matrix.platform }} (R ${{ matrix.r_version }})
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+
+      - name: Restore cached Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ matrix.platform }}-buildx-${{ github.sha }}
+          restore-keys: ${{ matrix.platform }}-buildx-
+
+      - name: Load cached Docker image
+        run: |
+          docker buildx build -t python-builds:${{ matrix.platform }} \
+            --file builder/Dockerfile.${{ matrix.platform }} \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --load \
+            builder
+
+      - name: Build Python
+        run: |
+          PYTHON_VERSION=${{ matrix.python_version }} make build-python-${{ matrix.platform }}
+
+      - name: Test Python
+        run: |
+          PYTHON_VERSION=${{ matrix.r_version }} make test-python-${{ matrix.platform }}

Makefile

@@ -43,6 +43,29 @@ serverless-deploy.%: deps fetch-serverless-custom-file
 break-glass.%: deps fetch-serverless-custom-file
 	$(SLS_BINARY) invoke stepf -n pythonBuilds -d '{"force": true}' --stage $*
 
+define GEN_TARGETS
+docker-build-$(platform):
+	@cd builder && docker-compose build $(platform)
+
+build-r-$(platform):
+	@cd builder && PYTHON_VERSION=$(PYTHON_VERSION) docker-compose run --rm $(platform)
+
+test-r-$(platform):
+	@cd test && PYTHON_VERSION=$(PYTHON_VERSION) docker-compose run --rm $(platform)
+
+bash-$(platform):
+	docker run -it --rm --entrypoint /bin/bash -v $(CURDIR):/python-builds python-builds:$(platform)
+
+.PHONY: docker-build-$(platform) build-python-$(platform) test-python-$(platform) bash-$(platform)
+endef
+
+$(foreach platform,$(PLATFORMS), \
+    $(eval $(GEN_TARGETS)) \
+)
+
+print-platforms:
+	@echo $(PLATFORMS)
+
 # Helper for launching a bash session on a docker image of your choice. Defaults
 # to "ubuntu:focal".
 TARGET_IMAGE?=ubuntu:focal

test/docker-compose.yml

@@ -0,0 +1,65 @@
+services:
+  ubuntu-2004:
+    image: ubuntu:focal
+    command: /python-builds/test/test-apt.sh
+    environment:
+      - OS_IDENTIFIER=ubuntu-2004
+      - PYTHON_VERSION=${PYTHON_VERSION}
+    volumes:
+      - ../:/python-builds
+  ubuntu-2204:
+    image: ubuntu:jammy
+    command: /python-builds/test/test-apt.sh
+    environment:
+      - OS_IDENTIFIER=ubuntu-2204
+      - PYTHON_VERSION=${PYTHON_VERSION}
+    volumes:
+      - ../:/python-builds
+  ubuntu-2404:
+    image: ubuntu:noble
+    command: /python-builds/test/test-apt.sh
+    environment:
+      - OS_IDENTIFIER=ubuntu-2404
+      - PYTHON_VERSION=${PYTHON_VERSION}
+    volumes:
+      - ../:/python-builds
+  debian-12:
+    image: debian:bookworm
+    command: /python-builds/test/test-apt.sh
+    environment:
+      - OS_IDENTIFIER=debian-12
+      - PYTHON_VERSION=${PYTHON_VERSION}
+    volumes:
+      - ../:/python-builds
+  debian-11:
+    image: debian:bullseye
+    command: /python-builds/test/test-apt.sh
+    environment:
+      - OS_IDENTIFIER=debian-11
+      - PYTHON_VERSION=${PYTHON_VERSION}
+    volumes:
+      - ../:/python-builds
+  centos-8:
+    image: rockylinux:8
+    command: /python-builds/test/test-yum.sh
+    environment:
+      - OS_IDENTIFIER=centos-8
+      - PYTHON_VERSION=${PYTHON_VERSION}
+    volumes:
+      - ../:/python-builds
+  rhel-9:
+    image: rockylinux:9
+    command: /python-builds/test/test-yum.sh
+    environment:
+      - OS_IDENTIFIER=rhel-9
+      - PYTHON_VERSION=${PYTHON_VERSION}
+    volumes:
+      - ../:/python-builds
+  opensuse-155:
+    image: opensuse/leap:15.5
+    command: /python-builds/test/test-zypper.sh
+    environment:
+      - OS_IDENTIFIER=opensuse-155
+      - PYTHON_VERSION=${PYTHON_VERSION}
+    volumes:
+      - ../:/python-builds

test/get_platforms.py

@@ -0,0 +1,31 @@
+import argparse
+import json
+import subprocess
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Print python-builds platforms as JSON.")
+    parser.add_argument(
+        'platforms',
+        type=str,
+        nargs='?',
+        default='all',
+        help='Comma-separated list of platforms. Specify "all" to use all platforms (the default).'
+    )
+    args = parser.parse_args()
+    platforms = _get_platforms(which=args.platforms)
+    print(json.dumps(platforms))
+
+
+def _get_platforms(which='all'):
+    supported_platforms = subprocess.check_output(['make', 'print-platforms'], text=True)
+    supported_platforms = supported_platforms.split()
+    if which == 'all':
+        return supported_platforms
+    platforms = which.split(',')
+    platforms = [p for p in platforms if p in supported_platforms]
+    return platforms
+
+
+if __name__ == '__main__':
+    main()

test/get_python_versions.py

@@ -0,0 +1,72 @@
+import argparse
+import json
+import re
+import urllib.request
+from packaging import version
+
+VERSIONS_URL = 'https://cdn.posit.co/python/versions.json'
+
+# Minimum Python version for "all"
+MIN_ALL_VERSION = version.parse('3.8.0')
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Print python-builds python versions as JSON.")
+    parser.add_argument(
+        'versions',
+        type=str,
+        nargs='?',
+        default='all',
+        help="""Comma-separated list of versions. Specify "last-N" to use the
+            last N minor python versions, or "all" to use all minor Python versions since 3.8.
+            Defaults to "last-5".
+            """
+    )
+    args = parser.parse_args()
+    versions = _get_versions(which=args.versions)
+    print(json.dumps(versions))
+
+
+def _get_versions(which='all'):
+    supported_versions = sorted(_get_supported_versions(), key=version.parse, reverse=True)
+    versions = []
+    for version_str in which.split(','):
+        versions.extend(_expand_version(version_str, supported_versions))
+    return versions
+
+def _expand_version(which, supported_versions):
+    last_n_versions = None
+    if which.startswith('last-'):
+        last_n_versions = int(which.replace('last-', ''))
+    elif which != 'all':
+        return [which] if which in supported_versions else []
+
+    versions = {}
+    for ver in supported_versions:
+        parsed_ver = version.parse(ver)
+        # Skip unreleased versions (e.g., devel, next)
+        if not re.match(r'[\d.]', ver):
+            continue
+        if parsed_ver < MIN_ALL_VERSION:
+            continue
+        minor_ver = (parsed_ver.major, parsed_ver.minor)
+        if minor_ver not in versions:
+            versions[minor_ver] = ver
+    versions = sorted(versions.values(), key=version.parse, reverse=True)
+
+    if last_n_versions:
+        return versions[0:last_n_versions]
+
+    return versions
+
+
+def _get_supported_versions():
+    request = urllib.request.Request(VERSIONS_URL)
+    response = urllib.request.urlopen(request)
+    data = response.read()
+    result = json.loads(data)
+    return result['python_versions']
+
+
+if __name__ == '__main__':
+    main()

test/test-apt.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -ex
+
+SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+# Install quick install script prerequisites
+if ! command -v curl > /dev/null 2>&1; then
+    apt update -qq
+    apt install -y curl
+fi
+
+# Run the quick install script. Use a locally built file if present, otherwise from the CDN.
+tmpdir=$(mktemp -d)
+cp -r "${SCRIPT_DIR}/../builder/integration/tmp/${OS_IDENTIFIER}/." "$tmpdir" > /dev/null 2>&1 || true
+(cd "$tmpdir" && SCRIPT_ACTION=install PYTHON_VERSION="${PYTHON_VERSION}" RUN_UNATTENDED=1 "${SCRIPT_DIR}/../install.sh")
+
+# Show DEB info
+apt show "python-${PYTHON_VERSION}"
+
+"${SCRIPT_DIR}/test-python.sh"
+
+apt remove -y "python-${PYTHON_VERSION}"
+
+if [ -d "/opt/python/${PYTHON_VERSION}" ]; then
+    echo "Failed to uninstall completely"
+    exit 1
+fi