diff --git a/packages/core/src/base/ActionsHandler.ts b/packages/core/src/base/ActionsHandler.ts
index 22f04491..e915f756 100644
--- a/packages/core/src/base/ActionsHandler.ts
+++ b/packages/core/src/base/ActionsHandler.ts
@@ -159,7 +159,10 @@ export default class ActionsHandler {
     }
 
     // only allow mouse left button
-    if (_eventType === EventType.Mouse && e.button !== MouseButton.Left) return
+    if (_eventType === EventType.Mouse && e.button !== MouseButton.Left) {
+      this.setInitiated()
+      return
+    }
 
     if (this.hooks.trigger(this.hooks.eventTypes.beforeStart, e)) {
       return