update_apps.yml

name: Update_Apps

# Controls when the workflow will run
on:
  # run at 00:00 and 12:00 UTC every Day
  schedule:
    - cron: "0 0,12 * * *"
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
# this workflow contains 3 jobs that run sequentially: get-latest-app-versions, test-updated-apps, and create-pr
# get-latest-app-versions gets the latests app versions and modifies the working directory of the repository with those changes. then it uploads that working directory of the git repository as an artifact for use in later jobs
# test-updated-apps runs a matrix of testcases where all updated apps from the first job are tested for successful installation/uninstallation in the working directory of the git repoistory downloaded from the artifact
# create-pr then uses the output from get-latest-app-versions and test-updated-apps to finalize a PR where only apps that passed all testcases are updated and any that fail have a descriptive error written in the PR comment
jobs:
  get-latest-app-versions:
    name: Get Latest App Versions
    # The type of runner that the job will run on
    runs-on: ubuntu-latest

    outputs:
      UPDATED_APPS: ${{ steps.updated.outputs.UPDATED_APPS }}
      FAILED_APPS: ${{ steps.failed.outputs.FAILED_APPS }}

    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v4

      # Runs a set of commands using the runners shell
      - name: Run app update scripts
        env:
          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
        run: |
          # print user info
          echo $USER $USERNAME $(id) $(whoami)
          
          #source pi-apps functions

          #export all functions and variables
          set -a
          #make DIRECTORY equal to GITHUB_WORKSPACE, for subscripts and api functions
          DIRECTORY=$GITHUB_WORKSPACE
          source $GITHUB_WORKSPACE/api

          #add special functions
          get_release() {
            curl -s --header "Authorization: token $GH_PERSONAL_ACCESS_TOKEN" "https://api.github.com/repos/$1/releases/latest" | jq -r '.tag_name' | sed s/^v//g
          }

          get_release_raw() {
            curl -s --header "Authorization: token $GH_PERSONAL_ACCESS_TOKEN" "https://api.github.com/repos/$1/releases/latest" | jq -r '.tag_name'
          }

          get_prerelease() {
            curl -s --header "Authorization: token $GH_PERSONAL_ACCESS_TOKEN" "https://api.github.com/repos/$1/releases" | jq -r 'map(select(.prerelease)) | first | .tag_name' | sed s/^v//g
          }

          get_prerelease_raw() {
            curl -s --header "Authorization: token $GH_PERSONAL_ACCESS_TOKEN" "https://api.github.com/repos/$1/releases" | jq -r 'map(select(.prerelease)) | first | .tag_name'
          }

          function validate_url(){
            if command wget --timeout=5 -q --spider "$1"; then
              return 0
            else
              return 1
            fi
          }

          #stop exporting functions
          set +a

          #make sure all update scripts are executable
          chmod +x $GITHUB_WORKSPACE/.github/workflows/updates/*.sh
          cd $GITHUB_WORKSPACE
          apps=( .github/workflows/updates/*.sh )
          for app_directory in "${apps[@]}"; do
            echo
            #make sure we are still in the main workspace (incase an update script left off elsewhere)
            cd $GITHUB_WORKSPACE
            export app_name="$(echo ${app_directory%.*} | sed 's:.*/::')"
            echo "$app_name"
            status "Checking $app_name for updates"
            # move to app folder
            cd "$GITHUB_WORKSPACE/apps/$app_name"
            # run app update script
            "$GITHUB_WORKSPACE/$app_directory"
          done
          cd

      - name: Output updated apps
        id: updated
        run: |
          if test -f /tmp/updated_apps; then
            sort -u /tmp/updated_apps > /tmp/updated_apps_sorted
            echo "UPDATED_APPS<<EOF" >> $GITHUB_OUTPUT
            cat /tmp/updated_apps_sorted >> $GITHUB_OUTPUT
            echo "EOF" >> $GITHUB_OUTPUT
          fi
      - name: Output failed apps
        id: failed
        run: |
          if test -f /tmp/failed_apps; then
            echo "FAILED_APPS<<EOF" >> $GITHUB_OUTPUT
            cat /tmp/failed_apps | sed '0~1 a\\' >> $GITHUB_OUTPUT
            echo "EOF" >> $GITHUB_OUTPUT
          fi
      - name: Tar GITHUB_WORKSPACE
        run: |
          cd ${{ runner.temp }}
          tar -C $GITHUB_WORKSPACE -cvf GITHUB_WORKSPACE.tar .

      - name: Upload GITHUB_WORKSPACE Artifact
        uses: actions/upload-artifact@v4
        with:
          name: GITHUB_WORKSPACE
          path: ${{ runner.temp }}/GITHUB_WORKSPACE.tar

  test-updated-apps:
    runs-on: ubuntu-latest
    needs: get-latest-app-versions
    strategy:
      matrix:
        include:
          - os: bullseye
            arch: armhf
            image: https://downloads.raspberrypi.org/raspios_oldstable_armhf/images/raspios_oldstable_armhf-2023-12-06/2023-12-05-raspios-bullseye-armhf.img.xz
            cache: yes
          - os: bullseye
            arch: arm64
            image: https://downloads.raspberrypi.org/raspios_oldstable_arm64/images/raspios_oldstable_arm64-2023-12-06/2023-12-05-raspios-bullseye-arm64.img.xz
            cache: yes
          - os: bookworm
            arch: armhf
            image: https://downloads.raspberrypi.com/raspios_armhf/images/raspios_armhf-2024-07-04/2024-07-04-raspios-bookworm-armhf.img.xz
            cache: yes
          - os: bookworm
            arch: arm64
            image: https://downloads.raspberrypi.com/raspios_arm64/images/raspios_arm64-2024-07-04/2024-07-04-raspios-bookworm-arm64.img.xz
            cache: yes
    name: Testing on ${{ matrix.os }} - ${{ matrix.arch }}
    outputs:
      FAILED_UPDATE_APPS_bullseye_armhf: ${{ steps.failed.outputs.FAILED_UPDATE_APPS_bullseye_armhf || '' }}
      FAILED_UPDATE_APPS_bullseye_arm64: ${{ steps.failed.outputs.FAILED_UPDATE_APPS_bullseye_arm64 || '' }}
      FAILED_UPDATE_APPS_bookworm_armhf: ${{ steps.failed.outputs.FAILED_UPDATE_APPS_bookworm_armhf || '' }}
      FAILED_UPDATE_APPS_bookworm_arm64: ${{ steps.failed.outputs.FAILED_UPDATE_APPS_bookworm_arm64 || '' }}
      UPDATED_APPS_bullseye_armhf: ${{ steps.updated.outputs.UPDATED_APPS_bullseye_armhf || '' }}
      UPDATED_APPS_bullseye_arm64: ${{ steps.updated.outputs.UPDATED_APPS_bullseye_arm64 || '' }}
      UPDATED_APPS_bookworm_armhf: ${{ steps.updated.outputs.UPDATED_APPS_bookworm_armhf || '' }}
      UPDATED_APPS_bookworm_arm64: ${{ steps.updated.outputs.UPDATED_APPS_bookworm_arm64 || '' }}
    steps:
      # restore GITHUB_WORKSPACE
      - uses: actions/download-artifact@v4
        with:
          name: GITHUB_WORKSPACE
          path: ${{ runner.temp }}
      - name: Extract GITHUB_WORKSPACE
        run: |
          cd ${{ runner.temp }}
          tar xf GITHUB_WORKSPACE.tar -C $GITHUB_WORKSPACE
          cd $GITHUB_WORKSPACE

      - name: Replace secret reference with contents
        if: ${{ matrix.image == 'RPIOS_TRIXIE_URL' || matrix.image == 'RPIOS_TRIXIE_ARMHF_URL' }}
        run: |
          image_url=${{ secrets[matrix.image] }}
          echo "::add-mask::$image_url"
          echo "image_url=$image_url" >> "$GITHUB_ENV"

      - name: Replace non-secret reference with contents
        if: ${{ matrix.image != 'RPIOS_TRIXIE_URL' && matrix.image != 'RPIOS_TRIXIE_ARMHF_URL' }}
        run: |
          image_url=${{ matrix.image }}
          echo "image_url=$image_url" >> "$GITHUB_ENV"

      - name: Write UPDATED_APPS and FAILED_APPS from OUTPUT of get-latest-app-versions to ENV
        run: |
          echo "UPDATED_APPS<<EOF" >> $GITHUB_ENV
          echo "${{needs.get-latest-app-versions.outputs.UPDATED_APPS}}" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

          echo "FAILED_APPS<<EOF" >> $GITHUB_ENV
          echo "${{needs.get-latest-app-versions.outputs.FAILED_APPS}}" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Test installing updated apps on ${{ matrix.os }} - ${{ matrix.arch }}
        uses: theofficialgman/arm-runner-action@v12
        with:
          base_image: ${{ env.image_url }}
          # bind mount the directory so any changes propogate to outside the chroot
          bind_mount_repository: yes

          # give the image more space
          image_additional_mb: 5000

          # disable image caching to not leak the URL on some systems
          enable_image_caching: ${{ matrix.cache }}

          # set CPUs to use
          cpu: cortex-a7:cortex-a72

          # use custom /proc/cpuinfo
          cpu_info: cpuinfo/raspberrypi_4b

          # user runner name as default path
          copy_repository_path: /home/runner/pi-apps

          # export github env back to outside the chroot
          export_github_env: yes

          import_github_env: true

          # disable debug which shows all commands to be executed
          debug: no

          # set shell to bash
          shell: /bin/bash
          commands: |
            sudo chown runner:docker /home/runner
            # print user info
            echo $USER $USERNAME $(id) $(whoami)
            sudo bash -c 'echo $USER $USERNAME $(id) $(whoami)'

            # create standard directories
            mkdir -p  $HOME/.local/share/applications $HOME/.local/bin
            sudo mkdir -p /usr/local/bin /usr/local/share/applications

            # install pi-apps dependencies
            sudo apt update
            sudo apt install -y yad curl wget aria2 lsb-release software-properties-common apt-utils imagemagick bc librsvg2-bin locales shellcheck git wmctrl xdotool x11-utils rsync

            # runonce-entries is run in the build tester, runonce requires that all api functions be available to subprocess (like is done in the gui script)
            #for the will_reinstall() and list_intersect() functions
            set -a #make all functions in the api available to subprocesses
            #make DIRECTORY equal to GITHUB_WORKSPACE, for subscripts and api functions
            DIRECTORY=$(pwd)
            source "${DIRECTORY}/api" || error "failed to source ${DIRECTORY}/api"
            #Run runonce entries
            "${DIRECTORY}/etc/runonce-entries"
            set +a #stop exporting functions
            cd ${DIRECTORY}

            # upgrade cmake to 3.20+ from theofficialgman ppa to fix QEMU only issue https://gitlab.kitware.com/cmake/cmake/-/issues/20568
            package_is_new_enough cmake 3.20 || debian_ppa_installer "theofficialgman/cmake-bionic" "bionic" "0ACACB5D1E74E484" || exit 1

            # store apps changed from last commit to working directory in variable
            mapfile -t changed_apps < <(git diff --name-only | grep ^apps/ | awk -F '/' '{print $2}' | sort -u)

            # clean out any app status files
            rm -rf ./data/status

            # attempt to install updated apps using manage script loop
            # if any app fails, add it to the FAILED_INSTALL_APPS or FAILED_UNINSTALL_APPS variable
            if [[ ${{ matrix.arch }} == armhf ]]; then
              # skip checking DDNet (due to Rust) if on arm32 under QEMU: https://github.com/rust-lang/cargo/issues/8719
              # skip checking Microsoft PowerShell due to QEMU instability with .NET: https://github.com/Botspot/pi-apps/pull/2252#issuecomment-1403043945
              # skip WPS Office on 32bit as it requires  "64bit" kernel to be detected. QEMU reports an armv7 kernel. WPS Office is still tested on arm64 (which will verify if the deb works and installs correctly)
              # skip Boxy SVG on 32bit due to QEMU instability
              skiplist="DDNet
            Microsoft PowerShell
            WPS Office
            Boxy SVG"
            else
              # skip checking Microsoft PowerShell due to QEMU instability with .NET: https://github.com/Botspot/pi-apps/pull/2252#issuecomment-1403043945
              skiplist="Microsoft PowerShell"
            fi            
            for app in "${changed_apps[@]}"; do
              # only attempt install/uninstall if updated app is supported on this architecture
              scriptname="$(script_name_cpu "$app")" #will be install, install-32, install-64, or package
              if [ -z "$scriptname" ];then
                continue
              fi
              # skip app if is in the skiplist
              if echo "$skiplist" | grep -Fxq "$app"; then
                continue
              fi
              ./manage install "$app" || { FAILED_INSTALL_APPS+="$app"$'\n'; UPDATED_APPS=$(echo "$UPDATED_APPS" | sed "/$app/d"); }
              ./manage uninstall "$app" || { FAILED_UNINSTALL_APPS+="$app"$'\n'; UPDATED_APPS=$(echo "$UPDATED_APPS" | sed "/$app/d"); }
            done

            if [ ! -z "$FAILED_INSTALL_APPS" ] || [ ! -z "$FAILED_UNINSTALL_APPS" ]; then
              echo "FAILED_UPDATE_APPS<<EOF" >> $GITHUB_ENV
              echo "$FAILED_INSTALL_APPS$FAILED_UNINSTALL_APPS" >> $GITHUB_ENV
              echo "EOF" >> $GITHUB_ENV
              echo "UPDATED_APPS<<EOF" >> $GITHUB_ENV
              echo "$UPDATED_APPS" >> $GITHUB_ENV
              echo "EOF" >> $GITHUB_ENV
            fi
      - name: Write FAILED_APPS to OUTPUT
        id: failed
        run: |
          echo "FAILED_UPDATE_APPS_${{ matrix.os }}_${{ matrix.arch }}<<EOF" >> $GITHUB_OUTPUT
          echo "${{env.FAILED_UPDATE_APPS}}" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT
      - name: Write UPDATED_APPS to OUTPUT
        id: updated
        run: |
          echo "UPDATED_APPS_${{ matrix.os }}_${{ matrix.arch }}<<EOF" >> $GITHUB_OUTPUT
          echo "${{env.UPDATED_APPS}}" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

  create-commit:
    needs: [get-latest-app-versions,test-updated-apps]
    runs-on: ubuntu-latest
    steps:
      # setup github environment variables
      - uses: actions/checkout@v4
      # restore GITHUB_WORKSPACE
      - uses: actions/download-artifact@v4
        with:
          name: GITHUB_WORKSPACE
          path: ${{ runner.temp }}
      - name: Extract GITHUB_WORKSPACE
        run: |
          cd ${{ runner.temp }}
          tar xf GITHUB_WORKSPACE.tar -C $GITHUB_WORKSPACE --exclude=".git"
          cd $GITHUB_WORKSPACE
      - name: Delete artifact
        uses: geekyeggo/delete-artifact@v5
        with:
          name: GITHUB_WORKSPACE
          failOnError: false

      - name: Revert failed apps and generate list
        run: |
          ALL_FAILED_APPS="${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bullseye_armhf}}"$'\n'"${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bullseye_arm64}}"$'\n'"${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bookworm_armhf}}"$'\n'"${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bookworm_arm64}}"
          ALL_FAILED_APPS="$(echo "$ALL_FAILED_APPS" | sort -u | awk NF)"
          echo "ALL_FAILED_APPS<<EOF" >> $GITHUB_ENV
          echo "$ALL_FAILED_APPS" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          error_string=""
          IFS=$'\n'
          for app in $ALL_FAILED_APPS; do
            git checkout -- "apps/$app"
            error_string+='![badge-error][badge-error]'" Failed to install $app on "

            if echo "${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bullseye_armhf}}" | grep -Fxq "$app"; then
              error_string+="bullseye armhf, "
            fi
            if echo "${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bullseye_arm64}}" | grep -Fxq "$app"; then
              error_string+="bullseye arm64, "
            fi
            if echo "${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bookworm_armhf}}" | grep -Fxq "$app"; then
              error_string+="bookworm armhf, "
            fi
            if echo "${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bookworm_arm64}}" | grep -Fxq "$app"; then
              error_string+="bookworm arm64, "
            fi

            error_string+="reverting to previous version."$'\n'$'\n'
          done
          true
          echo "ALL_FAILED_APPS_ERROR_STRING<<EOF" >> $GITHUB_ENV
          echo "$error_string" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Generate updated apps list
        run: |
          list_subtract() { #Outputs a list of apps from stdin, minus the ones that appear in $1 using the UPDATED_APP format
            # for example, the following two inputs will be a match
            # - BlockBench-arm64: 4.7.4 -> 4.8.0
            # BlockBench
            #                       change \n to -\n     | change '^' to '- '
            grep -ve "$(echo "$1" | sed -z 's/\n/\-\n/g' | sed 's/^/- /g')"
            true
          }
          ALL_UPDATED_APPS="${{needs.test-updated-apps.outputs.UPDATED_APPS_bullseye_armhf}}"$'\n'"${{needs.test-updated-apps.outputs.UPDATED_APPS_bullseye_arm64}}"$'\n'"${{needs.test-updated-apps.outputs.UPDATED_APPS_bookworm_armhf}}"$'\n'"${{needs.test-updated-apps.outputs.UPDATED_APPS_bookworm_arm64}}"
          ALL_UPDATED_APPS="$(echo "$ALL_UPDATED_APPS" | sort -u | awk NF)"
          [ ! -z "$ALL_UPDATED_APPS" ] && ALL_UPDATED_APPS="$(echo "$ALL_UPDATED_APPS" | list_subtract "$ALL_FAILED_APPS")"
          [ ! -z "$ALL_UPDATED_APPS" ] && COMMIT_TITLE="$(echo "$ALL_UPDATED_APPS" | awk -F '^- |-all:|-armhf:|-arm64:' '{print $2}' | sort -u | sed ':a;N;$!ba;s/\n/, /g' | sed -e 's/$/: Update App Versions/')"
          echo "ALL_UPDATED_APPS<<EOF" >> $GITHUB_ENV
          echo "$ALL_UPDATED_APPS" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          echo "COMMIT_TITLE<<EOF" >> $GITHUB_ENV
          echo "$COMMIT_TITLE" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Create Commit and Push
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          commit_author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
          commit_message: |
            ${{ env.COMMIT_TITLE }}

            ${{ env.ALL_UPDATED_APPS }}

      - name: Create or Update issue
        # don't error if this step fails
        # an upstream bug with no fix causes this to sometimes fail https://github.com/JasonEtco/create-an-issue/issues/142
        continue-on-error: true
        if: ${{ needs.get-latest-app-versions.outputs.FAILED_APPS != '' || env.ALL_FAILED_APPS_ERROR_STRING != '' }}
        uses: JasonEtco/create-an-issue@v2
        env:
          FAILED_APPS: ${{needs.get-latest-app-versions.outputs.FAILED_APPS}}
          ALL_FAILED_APPS_ERROR_STRING: ${{ env.ALL_FAILED_APPS_ERROR_STRING }}
          GITHUB_REPOSITORY: ${{ github.repository }}
          GITHUB_RUN_ID: ${{ github.run_id }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          update_existing: true
          search_existing: all
          filename: .github/update_apps_issue_template.md