Record XrSim tests for most of the sample projects (#170)

.gitattributes

@@ -7,5 +7,5 @@
 *.pro eol=lf
 *.properties eol=lf
 *.xml eol=lf
-
 gradlew eol=lf
+*.vrs filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-addon-on-push.yml

@@ -187,9 +187,15 @@ jobs:
    runs-on: [Windows, self-hosted, gpu]
    needs: build
 
+   env:
+     GODOT_VERSION: "4.3-beta2"
+     XRSIM_VERSION: "65.0.0"
+
    steps:
       - name: Checkout code
         uses: actions/checkout@v3
+        with:
+          lfs: 'true'
 
       - name: Install Chocolatey
         run: |
@@ -201,8 +207,7 @@ jobs:
 
       - name: Download Godot
         run: |
-          # @todo Download an actual Godot build!
-          Invoke-WebRequest -Uri "https://github.com/godotengine/godot-builds/releases/download/4.3-beta2/Godot_v4.3-beta2_win64.exe.zip" -OutFile "godot.zip"
+          Invoke-WebRequest -Uri "https://github.com/godotengine/godot-builds/releases/download/${{ env.GODOT_VERSION }}/Godot_v${{ env.GODOT_VERSION }}_win64.exe.zip" -OutFile "godot.zip"
 
       - name: Extract Godot
         run: |
@@ -214,7 +219,7 @@ jobs:
 
       - name: Download Meta XR Simulator
         run: |
-          Invoke-WebRequest -Uri "https://npm.developer.oculus.com/com.meta.xr.simulator/-/com.meta.xr.simulator-65.0.0.tgz" -OutFile MetaXRSimulator.tgz
+          Invoke-WebRequest -Uri "https://npm.developer.oculus.com/com.meta.xr.simulator/-/com.meta.xr.simulator-${{ env.XRSIM_VERSION }}.tgz" -OutFile MetaXRSimulator.tgz
 
       - name: Extract Meta XR Simulator
         run: |
@@ -223,14 +228,6 @@ jobs:
           tar -xzvf "MetaXRSimulator.tgz" -C "C:\Meta\"
           rm "MetaXRSimulator.tgz"
 
-          # Gunzip.
-          $tgzFile = Get-ChildItem -Path "C:\Meta" -Filter *.tgz
-          7z x $tgzFile.FullName -oC:\Meta -aoa
-
-          # Untar.
-          $tarFile = Get-ChildItem -Path "C:\Meta" -Filter *.tar
-          7z x $tarFile.FullName -oC:\Meta -aoa
-
       - name: Configure the Meta XR Simulator as the active OpenXR runtime
         run: |
           New-Item -Path "HKLM:\SOFTWARE\Khronos\OpenXR\1" -Name "ActiveRuntime" -Force
@@ -242,16 +239,38 @@ jobs:
           name: build-files-windows
           path: build-files-windows
 
-      - name: Copy Windows build of the addon into the demo project
+      - name: Copy Windows build of the addon into the demo and samples project
         run: |
           mkdir -p demo/addons/godotopenxrvendors/.bin/windows/
           cp -r build-files-windows/* demo/addons/godotopenxrvendors/.bin/windows/
 
-      - name: Import the demo project
+          $sampleProjects = Get-ChildItem -Path "samples/" -Directory
+          foreach ($project in $sampleProjects) {
+            cp -r demo/addons "$($project.FullName)/"
+          }
+
+      - name: Import the demo and sample projects
         run: |
-          $godot = "Godot_v4.3-beta2_win64.exe"
+          $godot = "Godot_v${{ env.GODOT_VERSION }}_win64.exe"
+
+          Write-Output ""
+          Write-Output " ** Importing demo project"
+          Write-Output ""
+
           Start-Process -FilePath "$godot" -ArgumentList "--path demo --import --headless" -NoNewWindow -Wait
 
+          $sampleProjects = Get-ChildItem -Path "samples/" -Directory
+          foreach ($project in $sampleProjects) {
+            $projectFile = Join-Path -Path $project.FullName -ChildPath "project.godot"
+              if (Test-Path $projectFile) {
+              Write-Output ""
+              Write-Output " ** Importing project $($project.Name)"
+              Write-Output ""
+
+              Start-Process -FilePath "$godot" -ArgumentList "--path $($project.FullName) --import --headless" -NoNewWindow -Wait
+            }
+          }
+
       - name: Launch a synthetic environment
         run: |
           # Ensure a synthetic environment isn't already running.
@@ -267,7 +286,12 @@ jobs:
         run: |
           $jsonPath = "$env:AppData\MetaXR\MetaXrSimulator\persistent_data.json"
 
-          $vrsFiles = Get-ChildItem -Path tests/vrs -Filter *.vrs
+          $demoVrsFiles = Get-ChildItem -Path "demo/tests/" -Recurse -Filter *.vrs
+          $sampleVrsFiles = Get-ChildItem -Path "samples/*/tests/" -Recurse -Filter *.vrs
+          $vrsFiles = @()
+          $vrsFiles += $demoVrsFiles
+          $vrsFiles += $sampleVrsFiles
+
           foreach ($file in $vrsFiles) {
             $replayPath = Join-Path -Path $file.DirectoryName -ChildPath ($file.BaseName + "-replay.vrs")
             $jsonContent = @{
@@ -285,11 +309,17 @@ jobs:
             Set-Content -Path $jsonPath -Value $jsonContent
             echo "$jsonContent"
 
-            $godot = "Godot_v4.3-beta2_win64.exe"
+            $godot = "Godot_v${{ env.GODOT_VERSION }}_win64.exe"
             $timeout = 300
             $waitTime = 0
 
-            $process = Start-Process -FilePath "$godot" -ArgumentList "--path demo --rendering-method mobile --verbose -- --quit-with-openxr" -NoNewWindow -PassThru
+            $projectPath = (Get-Item $file.DirectoryName).Parent
+
+            Write-Output ""
+            Write-Output " ** Running VRS $($file.Name) on $($projectPath.Name) project"
+            Write-Output ""
+
+            $process = Start-Process -FilePath "$godot" -ArgumentList "--path $($projectPath.FullName) --rendering-method mobile --verbose -- --xrsim-automated-tests" -NoNewWindow -PassThru
 
             while ($process.HasExited -eq $false -and $waitTime -lt $timeout) {
                 Start-Sleep -Seconds 1
@@ -314,17 +344,13 @@ jobs:
         run: |
           Get-Process -Name "synth_env_server" | Stop-Process
 
-      - name: Copy vrs_pixmatch.py from Meta XR Simulator
-        run: |
-          $scriptsPath = "C:\Meta\package\MetaXRSimulator\scripts"
-          cp "$scriptsPath\requirements.txt" tests\vrs\
-          cp "$scriptsPath\vrs_pixmatch.py" tests\vrs\
-
       - name: Upload VRS artifacts
         uses: actions/upload-artifact@v3
         with:
           name: ReplayVRS
-          path: tests/vrs/**/*
+          path: |
+            test-scripts/
+            **/*/tests/*.vrs
 
   compare_vrs_replay:
     name: "Compare VRS replay"
@@ -349,23 +375,24 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r tests/vrs/requirements.txt
+          pip install -r scripts/vrs_pixmatch/requirements.txt
 
       - name: Compare VRS replay with expected recordings
         run: |
           cd tests/vrs/
 
-          # Fix bugs in vrs_pixmatch.py script.
-          patch -p0 < ../../thirdparty/meta_xr_simulator/vrs_pixmatch.patch
-
           mkdir diffs
-          for replay in *-replay.vrs; do
+          for replay in $(find . -name \*-replay.vrs); do
+            echo " ==> Checking $replay..."
             expected=$(echo $replay | sed -e 's/-replay.vrs$/.vrs/')
-            python vrs_pixmatch.py "$replay" "$expected" --threshold 0.4 --best_match_pixels_diff_threshold 40000 --diffs_output_path diffs
+            diff_path="diffs/$replay"
+            mkdir -p "$diff_path"
+            python ../../scripts/vrs_pixmatch/vrs_pixmatch.py "$expected" "$replay" --threshold 0.4 --max-pixel-diff 40000 --output-path "$diff_path" --verbose
           done
 
       - name: Upload VRS diff artifacts
         uses: actions/upload-artifact@v3
+        if: always()
         with:
           name: ReplayVRSDiff
           path: tests/vrs/diffs/**/*

demo/main.gd

@@ -24,7 +24,7 @@ func _ready():
 
 
 func _on_session_stopping() -> void:
-	if "--quit-with-openxr" in OS.get_cmdline_user_args():
+	if "--xrsim-automated-tests" in OS.get_cmdline_user_args():
 		# When we're running tests via the XR Simulator, it will end the OpenXR
 		# session automatically, and in that case, we want to quit.
 		get_tree().quit()

samples/meta-body-tracking-sample/start_xr.gd

@@ -107,7 +107,7 @@ func _on_openxr_stopping() -> void:
 	# Our session is being stopped.
 	print("OpenXR is stopping")
 
-	if "--quit-with-openxr" in OS.get_cmdline_user_args():
+	if "--xrsim-automated-tests" in OS.get_cmdline_user_args():
 		# When we're running tests via the XR Simulator, it will end the OpenXR
 		# session automatically, and in that case, we want to quit.
 		get_tree().quit()

samples/meta-composition-layers-sample/example_panel.gd

@@ -12,7 +12,10 @@ func _ready() -> void:
 	$Label.text = display_string
 
 func _process(delta: float) -> void:
-	$Label.position = Vector2(start_pos.x + (100.0 * sin(Time.get_ticks_msec() * 0.001 * 2)), start_pos.y)
+	if not "--xrsim-automated-tests" in OS.get_cmdline_user_args():
+		# When we're running tests via the XR Simulator, we don't want the text
+		# to be animated, which can lead to differences in the screenshots.
+		$Label.position = Vector2(start_pos.x + (100.0 * sin(Time.get_ticks_msec() * 0.001 * 2)), start_pos.y)
 
 func update_pointers(p_position_left: Vector2, p_position_right: Vector2) -> void:
 	pointer_left.position = size * p_position_left - (0.5 * pointer_left.size)

samples/meta-composition-layers-sample/start_xr.gd

@@ -107,7 +107,7 @@ func _on_openxr_stopping() -> void:
 	# Our session is being stopped.
 	print("OpenXR is stopping")
 
-	if "--quit-with-openxr" in OS.get_cmdline_user_args():
+	if "--xrsim-automated-tests" in OS.get_cmdline_user_args():
 		# When we're running tests via the XR Simulator, it will end the OpenXR
 		# session automatically, and in that case, we want to quit.
 		get_tree().quit()

samples/meta-hand-tracking-sample/start_xr.gd

@@ -107,7 +107,7 @@ func _on_openxr_stopping() -> void:
 	# Our session is being stopped.
 	print("OpenXR is stopping")
 
-	if "--quit-with-openxr" in OS.get_cmdline_user_args():
+	if "--xrsim-automated-tests" in OS.get_cmdline_user_args():
 		# When we're running tests via the XR Simulator, it will end the OpenXR
 		# session automatically, and in that case, we want to quit.
 		get_tree().quit()

samples/meta-passthrough-sample/geometry.gd

@@ -2,4 +2,7 @@ extends OpenXRFbPassthroughGeometry
 
 
 func _process(delta: float) -> void:
-	position.x = 3 * sin(Time.get_ticks_msec() / 1000.0)
+	if not "--xrsim-automated-tests" in OS.get_cmdline_user_args():
+		# When we're running tests via the XR Simulator, we don't want this
+		# to be animated, which can lead to differences in the screenshots.
+		position.x = 3 * sin(Time.get_ticks_msec() / 1000.0)

samples/meta-passthrough-sample/start_xr.gd

@@ -107,7 +107,7 @@ func _on_openxr_stopping() -> void:
 	# Our session is being stopped.
 	print("OpenXR is stopping")
 
-	if "--quit-with-openxr" in OS.get_cmdline_user_args():
+	if "--xrsim-automated-tests" in OS.get_cmdline_user_args():
 		# When we're running tests via the XR Simulator, it will end the OpenXR
 		# session automatically, and in that case, we want to quit.
 		get_tree().quit()

samples/meta-scene-sample/spatial_anchor.gd

@@ -14,7 +14,6 @@ func setup_scene(spatial_entity: OpenXRFbSpatialEntity) -> void:
 	material.albedo_color = color
 	mesh_instance.set_surface_override_material(0, material)
 
-	spatial_entity.openxr_fb_spatial_entity_saved.connect(self._on_saved_to_cloud.bind(spatial_entity))
 	spatial_entity.save_to_storage(OpenXRFbSpatialEntity.STORAGE_CLOUD)
 
 func set_selected(p_selected: bool) -> void:

samples/meta-scene-sample/start_xr.gd

@@ -107,7 +107,7 @@ func _on_openxr_stopping() -> void:
 	# Our session is being stopped.
 	print("OpenXR is stopping")
 
-	if "--quit-with-openxr" in OS.get_cmdline_user_args():
+	if "--xrsim-automated-tests" in OS.get_cmdline_user_args():
 		# When we're running tests via the XR Simulator, it will end the OpenXR
 		# session automatically, and in that case, we want to quit.
 		get_tree().quit()

scripts/vrs_pixmatch/requirements.txt

@@ -0,0 +1,20 @@
+contourpy==1.1.1
+cycler==0.12.1
+dataclasses==0.6
+fonttools==4.47.2
+importlib-resources==6.1.1
+kiwisolver==1.4.5
+matplotlib==3.7.4
+numpy==1.24.4
+packaging==23.2
+pillow==10.2.0
+pixelmatch==0.3.0
+pyparsing==3.1.1
+PyQt5==5.15.10
+PyQt5-Qt5==5.15.14
+PyQt5-sip==12.13.0
+python-dateutil==2.8.2
+six==1.16.0
+typing==3.7.4.3
+vrs==1.0.4
+zipp==3.17.0