diff --git a/Frontend/Docs/Resources/Images/settings-panel.png b/Frontend/Docs/Resources/Images/settings-panel.png index ff16004b..521bb263 100644 Binary files a/Frontend/Docs/Resources/Images/settings-panel.png and b/Frontend/Docs/Resources/Images/settings-panel.png differ diff --git a/Frontend/Docs/Settings Panel.md b/Frontend/Docs/Settings Panel.md index b510a0b8..b2705d62 100644 --- a/Frontend/Docs/Settings Panel.md +++ b/Frontend/Docs/Settings Panel.md @@ -35,6 +35,15 @@ This page will be updated with new features and commands as they become availabl | **Control scheme** | If the scheme is `locked mouse` the browser will use `pointerlock` to capture your mouse, whereas if the scheme is `hovering mouse` you will retain your OS/browser cursor. | | **Color scheme** | Allows you to switch between light mode and dark mode. | +### Input +| **Setting** | **Description** | +| --- | --- | +| **Keyboard input** | If enabled, captures and sends keyboard events to the Unreal Engine application. | +| **Mouse input** | If enabled, captures and sends mouse events to the Unreal Engine application. | +| **Touch input** | If enabled, captures and sends touch events to the Unreal Engine application. | +| **Gamepad input** | If enabled, captures and sends gamepad events to the Unreal Engine application. | +| **XR controller input** | If enabled, captures and sends XR controller events to the Unreal Engine application. | + ### Encoder | **Setting** | **Description** | | --- | --- | diff --git a/Frontend/library/package-lock.json b/Frontend/library/package-lock.json index 9929e2d8..634a3e60 100644 --- a/Frontend/library/package-lock.json +++ b/Frontend/library/package-lock.json @@ -1,12 +1,12 @@ { "name": "@epicgames-ps/lib-pixelstreamingfrontend-ue5.2", - "version": "0.1.1", + "version": "0.1.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@epicgames-ps/lib-pixelstreamingfrontend-ue5.2", - "version": "0.1.1", + "version": "0.1.3", "license": "MIT", "dependencies": { "sdp": "^3.1.0" diff --git a/Frontend/library/src/Config/Config.ts b/Frontend/library/src/Config/Config.ts index 200c3d77..6f64fc82 100644 --- a/Frontend/library/src/Config/Config.ts +++ b/Frontend/library/src/Config/Config.ts @@ -27,6 +27,11 @@ export class Flags { static StartVideoMuted = 'StartVideoMuted' as const; static SuppressBrowserKeys = 'SuppressBrowserKeys' as const; static UseMic = 'UseMic' as const; + static KeyboardInput = 'KeyboardInput' as const; + static MouseInput = 'MouseInput' as const; + static TouchInput = 'TouchInput' as const; + static GamepadInput = 'GamepadInput' as const; + static XRControllerInput = 'XRControllerInput' as const; } export type FlagsKeys = Exclude; @@ -398,6 +403,61 @@ export class Config { ) ); + this.flags.set( + Flags.KeyboardInput, + new SettingFlag( + Flags.KeyboardInput, + 'Keyboard input', + 'If enabled, send keyboard events to streamer', + true, + useUrlParams + ) + ); + + this.flags.set( + Flags.MouseInput, + new SettingFlag( + Flags.MouseInput, + 'Mouse input', + 'If enabled, send mouse events to streamer', + true, + useUrlParams + ) + ); + + this.flags.set( + Flags.TouchInput, + new SettingFlag( + Flags.TouchInput, + 'Touch input', + 'If enabled, send touch events to streamer', + true, + useUrlParams + ) + ); + + this.flags.set( + Flags.GamepadInput, + new SettingFlag( + Flags.GamepadInput, + 'Gamepad input', + 'If enabled, send gamepad events to streamer', + true, + useUrlParams + ) + ); + + this.flags.set( + Flags.XRControllerInput, + new SettingFlag( + Flags.XRControllerInput, + 'XR controller input', + 'If enabled, send XR controller events to streamer', + true, + useUrlParams + ) + ); + /** * Numeric parameters */ diff --git a/Frontend/library/src/Inputs/FakeTouchController.ts b/Frontend/library/src/Inputs/FakeTouchController.ts index 982a2f2d..21d63ad9 100644 --- a/Frontend/library/src/Inputs/FakeTouchController.ts +++ b/Frontend/library/src/Inputs/FakeTouchController.ts @@ -5,6 +5,7 @@ import { StreamMessageController } from '../UeInstanceMessage/StreamMessageContr import { VideoPlayer } from '../VideoPlayer/VideoPlayer'; import { ITouchController } from './ITouchController'; import { MouseButton } from './MouseButtons'; +import { EventListenerTracker } from '../Util/EventListenerTracker'; /** * Allows for the usage of fake touch events and implements ITouchController @@ -18,6 +19,9 @@ export class FakeTouchController implements ITouchController { coordinateConverter: CoordinateConverter; videoElementParentClientRect: DOMRect; + // Utility for keeping track of event handlers and unregistering them + private touchEventListenerTracker = new EventListenerTracker(); + /** * @param toStreamerMessagesProvider - Stream message instance * @param videoElementProvider - Video element instance @@ -31,9 +35,28 @@ export class FakeTouchController implements ITouchController { this.toStreamerMessagesProvider = toStreamerMessagesProvider; this.videoElementProvider = videoElementProvider; this.coordinateConverter = coordinateConverter; - document.ontouchstart = (ev: TouchEvent) => this.onTouchStart(ev); - document.ontouchend = (ev: TouchEvent) => this.onTouchEnd(ev); - document.ontouchmove = (ev: TouchEvent) => this.onTouchMove(ev); + const ontouchstart = (ev: TouchEvent) => this.onTouchStart(ev); + const ontouchend = (ev: TouchEvent) => this.onTouchEnd(ev); + const ontouchmove = (ev: TouchEvent) => this.onTouchMove(ev); + document.addEventListener('touchstart', ontouchstart, { passive: false }); + document.addEventListener('touchend', ontouchend, { passive: false }); + document.addEventListener('touchmove', ontouchmove, { passive: false }); + this.touchEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener('touchstart', ontouchstart) + ); + this.touchEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener('touchend', ontouchend) + ); + this.touchEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener('touchmove', ontouchmove) + ); + } + + /** + * Unregister all touch events + */ + unregisterTouchEvents() { + this.touchEventListenerTracker.unregisterAll(); } /** @@ -62,8 +85,8 @@ export class FakeTouchController implements ITouchController { const videoElementParent = this.videoElementProvider.getVideoParentElement() as HTMLDivElement; - const mouseEvent = new MouseEvent(touch.type, first_touch); - videoElementParent.onmouseenter(mouseEvent); + const mouseEvent = new MouseEvent('mouseenter', first_touch); + videoElementParent.dispatchEvent(mouseEvent); const coord = this.coordinateConverter.normalizeAndQuantizeUnsigned( this.fakeTouchFinger.x, @@ -107,8 +130,8 @@ export class FakeTouchController implements ITouchController { coord.y ]); - const mouseEvent = new MouseEvent(touchEvent.type, touch); - videoElementParent.onmouseleave(mouseEvent); + const mouseEvent = new MouseEvent('mouseleave', touch); + videoElementParent.dispatchEvent(mouseEvent); this.fakeTouchFinger = null; break; } @@ -140,7 +163,7 @@ export class FakeTouchController implements ITouchController { x - this.fakeTouchFinger.x, y - this.fakeTouchFinger.y ); - toStreamerHandlers.get('MoveMouse')([ + toStreamerHandlers.get('MouseMove')([ coord.x, coord.y, delta.x, diff --git a/Frontend/library/src/Inputs/GamepadController.ts b/Frontend/library/src/Inputs/GamepadController.ts index cdba4bbb..9eaf7ae5 100644 --- a/Frontend/library/src/Inputs/GamepadController.ts +++ b/Frontend/library/src/Inputs/GamepadController.ts @@ -2,6 +2,7 @@ import { Logger } from '../Logger/Logger'; import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController'; +import { EventListenerTracker } from '../Util/EventListenerTracker'; import { Controller } from './GamepadTypes'; /** @@ -12,35 +13,60 @@ export class GamePadController { requestAnimationFrame: (callback: FrameRequestCallback) => number; toStreamerMessagesProvider: StreamMessageController; + // Utility for keeping track of event handlers and unregistering them + private gamePadEventListenerTracker = new EventListenerTracker(); + /** * @param toStreamerMessagesProvider - Stream message instance */ constructor(toStreamerMessagesProvider: StreamMessageController) { this.toStreamerMessagesProvider = toStreamerMessagesProvider; - this.requestAnimationFrame = + this.requestAnimationFrame = ( window.mozRequestAnimationFrame || window.webkitRequestAnimationFrame || - window.requestAnimationFrame; + window.requestAnimationFrame + ).bind(window); const browserWindow = window as Window; if ('GamepadEvent' in browserWindow) { - window.addEventListener('gamepadconnected', (ev: GamepadEvent) => - this.gamePadConnectHandler(ev) + const onGamePadConnected = (ev: GamepadEvent) => + this.gamePadConnectHandler(ev); + const onGamePadDisconnected = (ev: GamepadEvent) => + this.gamePadDisconnectHandler(ev); + window.addEventListener('gamepadconnected', onGamePadConnected); + window.addEventListener('gamepaddisconnected', onGamePadDisconnected); + this.gamePadEventListenerTracker.addUnregisterCallback( + () => window.removeEventListener('gamepadconnected', onGamePadConnected) ); - window.addEventListener('gamepaddisconnected', (ev: GamepadEvent) => - this.gamePadDisconnectHandler(ev) + this.gamePadEventListenerTracker.addUnregisterCallback( + () => window.removeEventListener('gamepaddisconnected', onGamePadDisconnected) ); } else if ('WebKitGamepadEvent' in browserWindow) { - window.addEventListener( - 'webkitgamepadconnected', - (ev: GamepadEvent) => this.gamePadConnectHandler(ev) + const onWebkitGamePadConnected = (ev: GamepadEvent) => this.gamePadConnectHandler(ev); + const onWebkitGamePadDisconnected = (ev: GamepadEvent) => this.gamePadDisconnectHandler(ev); + window.addEventListener('webkitgamepadconnected', onWebkitGamePadConnected); + window.addEventListener('webkitgamepaddisconnected', onWebkitGamePadDisconnected); + this.gamePadEventListenerTracker.addUnregisterCallback( + () => window.removeEventListener('webkitgamepadconnected', onWebkitGamePadConnected) ); - window.addEventListener( - 'webkitgamepaddisconnected', - (ev: GamepadEvent) => this.gamePadDisconnectHandler(ev) + this.gamePadEventListenerTracker.addUnregisterCallback( + () => window.removeEventListener('webkitgamepaddisconnected', onWebkitGamePadDisconnected) ); } this.controllers = []; + for (const gamepad of navigator.getGamepads()) { + if (gamepad) { + this.gamePadConnectHandler(new GamepadEvent('gamepadconnected', { gamepad })); + } + } + } + + /** + * Unregisters all event handlers + */ + unregisterGamePadEvents() { + this.gamePadEventListenerTracker.unregisterAll(); + this.controllers = []; } /** @@ -64,7 +90,7 @@ export class GamePadController { 'gamepad: ' + gamepad.id + ' connected', 6 ); - window.requestAnimationFrame(() => this.updateStatus()); + this.requestAnimationFrame(() => this.updateStatus()); } /** @@ -185,7 +211,9 @@ export class GamePadController { } this.controllers[controllerIndex].prevState = currentState; } - this.requestAnimationFrame(() => this.updateStatus()); + if (this.controllers.length > 0) { + this.requestAnimationFrame(() => this.updateStatus()); + } } } diff --git a/Frontend/library/src/Inputs/HoveringMouseEvents.ts b/Frontend/library/src/Inputs/HoveringMouseEvents.ts index 674eafa5..6526dffd 100644 --- a/Frontend/library/src/Inputs/HoveringMouseEvents.ts +++ b/Frontend/library/src/Inputs/HoveringMouseEvents.ts @@ -17,6 +17,13 @@ export class HoveringMouseEvents implements IMouseEvents { this.mouseController = mouseController; } + /** + * Unregister event handlers + */ + unregisterMouseEvents(): void { + // empty for HoveringMouseEvents implementation + } + /** * Handle the mouse move event, sends the mouse data to the UE Instance * @param mouseEvent - Mouse Event diff --git a/Frontend/library/src/Inputs/IMouseEvents.ts b/Frontend/library/src/Inputs/IMouseEvents.ts index 7c9033cd..65d2dc65 100644 --- a/Frontend/library/src/Inputs/IMouseEvents.ts +++ b/Frontend/library/src/Inputs/IMouseEvents.ts @@ -56,4 +56,9 @@ export interface IMouseEvents { * @param mouseEvent - mouse event */ handleContextMenu?(mouseEvent: MouseEvent): void; + + /** + * Unregisters any registered mouse event handlers + */ + unregisterMouseEvents(): void; } diff --git a/Frontend/library/src/Inputs/ITouchController.ts b/Frontend/library/src/Inputs/ITouchController.ts index 39cb0396..c8f9d562 100644 --- a/Frontend/library/src/Inputs/ITouchController.ts +++ b/Frontend/library/src/Inputs/ITouchController.ts @@ -21,4 +21,9 @@ export interface ITouchController { * @param touchEvent - Touch Event Data */ onTouchMove(touchEvent: TouchEvent): void; + + /** + * Unregisters all touch event handlers + */ + unregisterTouchEvents(): void; } diff --git a/Frontend/library/src/Inputs/InputClassesFactory.ts b/Frontend/library/src/Inputs/InputClassesFactory.ts index c95b2177..74076193 100644 --- a/Frontend/library/src/Inputs/InputClassesFactory.ts +++ b/Frontend/library/src/Inputs/InputClassesFactory.ts @@ -6,9 +6,6 @@ import { MouseController } from './MouseController'; import { TouchController } from './TouchController'; import { GamePadController } from './GamepadController'; import { Config, ControlSchemeType } from '../Config/Config'; -import { LockedMouseEvents } from './LockedMouseEvents'; -import { HoveringMouseEvents } from './HoveringMouseEvents'; -import { IMouseEvents } from './IMouseEvents'; import { Logger } from '../Logger/Logger'; import { CoordinateConverter } from '../Util/CoordinateConverter'; import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController'; @@ -61,110 +58,29 @@ export class InputClassesFactory { const mouseController = new MouseController( this.toStreamerMessagesProvider, this.videoElementProvider, - this.coordinateConverter + this.coordinateConverter, + this.activeKeys ); - mouseController.clearMouseEvents(); switch (controlScheme) { case ControlSchemeType.LockedMouse: - this.registerLockedMouseEvents(mouseController); + mouseController.registerLockedMouseEvents(mouseController); break; case ControlSchemeType.HoveringMouse: - this.registerHoveringMouseEvents(mouseController); + mouseController.registerHoveringMouseEvents(mouseController); break; default: Logger.Info( Logger.GetStackTrace(), 'unknown Control Scheme Type Defaulting to Locked Mouse Events' ); - this.registerLockedMouseEvents(mouseController); + mouseController.registerLockedMouseEvents(mouseController); break; } return mouseController; } - /** - * Register a locked mouse class - * @param mouseController - a mouse controller instance - * @param playerStyleAttributesProvider - a player style attributes instance - */ - registerLockedMouseEvents(mouseController: MouseController) { - const videoElementParent = - this.videoElementProvider.getVideoParentElement() as HTMLDivElement; - const lockedMouseEvents: IMouseEvents = new LockedMouseEvents( - this.videoElementProvider, - mouseController, - this.activeKeys - ); - - videoElementParent.requestPointerLock = - videoElementParent.requestPointerLock || - videoElementParent.mozRequestPointerLock; - document.exitPointerLock = - document.exitPointerLock || document.mozExitPointerLock; - - // minor hack to alleviate ios not supporting pointerlock - if (videoElementParent.requestPointerLock) { - videoElementParent.onclick = () => { - videoElementParent.requestPointerLock(); - }; - } - - const lockStateChangeListener = () => - lockedMouseEvents.lockStateChange(); - document.addEventListener( - 'pointerlockchange', - lockStateChangeListener, - false - ); - document.addEventListener( - 'mozpointerlockchange', - lockStateChangeListener, - false - ); - - videoElementParent.onmousedown = (mouseEvent: MouseEvent) => - lockedMouseEvents.handleMouseDown(mouseEvent); - videoElementParent.onmouseup = (mouseEvent: MouseEvent) => - lockedMouseEvents.handleMouseUp(mouseEvent); - videoElementParent.onwheel = (wheelEvent: WheelEvent) => - lockedMouseEvents.handleMouseWheel(wheelEvent); - videoElementParent.ondblclick = (mouseEvent: MouseEvent) => - lockedMouseEvents.handleMouseDouble(mouseEvent); - videoElementParent.pressMouseButtons = (mouseEvent: MouseEvent) => - lockedMouseEvents.handlePressMouseButtons(mouseEvent); - videoElementParent.releaseMouseButtons = (mouseEvent: MouseEvent) => - lockedMouseEvents.handleReleaseMouseButtons(mouseEvent); - } - - /** - * Register a hovering mouse class - * @param mouseController - A mouse controller object - */ - registerHoveringMouseEvents(mouseController: MouseController) { - const videoElementParent = - this.videoElementProvider.getVideoParentElement() as HTMLDivElement; - const hoveringMouseEvents = new HoveringMouseEvents(mouseController); - - videoElementParent.onmousemove = (mouseEvent: MouseEvent) => - hoveringMouseEvents.updateMouseMovePosition(mouseEvent); - videoElementParent.onmousedown = (mouseEvent: MouseEvent) => - hoveringMouseEvents.handleMouseDown(mouseEvent); - videoElementParent.onmouseup = (mouseEvent: MouseEvent) => - hoveringMouseEvents.handleMouseUp(mouseEvent); - videoElementParent.oncontextmenu = (mouseEvent: MouseEvent) => - hoveringMouseEvents.handleContextMenu(mouseEvent); - videoElementParent.onwheel = (wheelEvent: WheelEvent) => - hoveringMouseEvents.handleMouseWheel(wheelEvent); - videoElementParent.ondblclick = (mouseEvent: MouseEvent) => - hoveringMouseEvents.handleMouseDouble(mouseEvent); - videoElementParent.pressMouseButtons = (mouseEvent: MouseEvent) => - hoveringMouseEvents.handlePressMouseButtons(mouseEvent); - videoElementParent.releaseMouseButtons = (mouseEvent: MouseEvent) => - hoveringMouseEvents.handleReleaseMouseButtons(mouseEvent); - } - /** * register touch events * @param fakeMouseTouch - the faked mouse touch event diff --git a/Frontend/library/src/Inputs/KeyboardController.ts b/Frontend/library/src/Inputs/KeyboardController.ts index 6866c045..f650a832 100644 --- a/Frontend/library/src/Inputs/KeyboardController.ts +++ b/Frontend/library/src/Inputs/KeyboardController.ts @@ -5,6 +5,7 @@ import { Logger } from '../Logger/Logger'; import { ActiveKeys } from './InputClassesFactory'; import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController'; import { Config, Flags } from '../Config/Config'; +import { EventListenerTracker } from '../Util/EventListenerTracker'; interface ICodeToKeyCode { [key: string]: number; @@ -18,6 +19,9 @@ export class KeyboardController { activeKeysProvider: ActiveKeys; config: Config; + // Utility for keeping track of event handlers and unregistering them + private keyboardEventListenerTracker = new EventListenerTracker(); + /* * New browser APIs have moved away from KeyboardEvent.keyCode to KeyboardEvent.Code. * For details see: https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/keyCode#constants_for_keycode_value @@ -146,11 +150,32 @@ export class KeyboardController { * Registers document keyboard events with the controller */ registerKeyBoardEvents() { - document.onkeydown = (ev: KeyboardEvent) => this.handleOnKeyDown(ev); - document.onkeyup = (ev: KeyboardEvent) => this.handleOnKeyUp(ev); + const keyDownHandler = (ev: KeyboardEvent) => this.handleOnKeyDown(ev); + const keyUpHandler = (ev: KeyboardEvent) => this.handleOnKeyUp(ev); + const keyPressHandler = (ev: KeyboardEvent) => this.handleOnKeyPress(ev); + + document.addEventListener("keydown", keyDownHandler); + document.addEventListener("keyup", keyUpHandler); //This has been deprecated as at Jun 13 2021 - document.onkeypress = (ev: KeyboardEvent) => this.handleOnKeyPress(ev); + document.addEventListener("keypress", keyPressHandler); + + this.keyboardEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener("keydown", keyDownHandler) + ); + this.keyboardEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener("keyup", keyUpHandler) + ); + this.keyboardEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener("keypress", keyPressHandler) + ); + } + + /** + * Unregisters document keyboard events + */ + unregisterKeyBoardEvents() { + this.keyboardEventListenerTracker.unregisterAll(); } /** diff --git a/Frontend/library/src/Inputs/LockedMouseEvents.ts b/Frontend/library/src/Inputs/LockedMouseEvents.ts index 38e3af6b..f374eee7 100644 --- a/Frontend/library/src/Inputs/LockedMouseEvents.ts +++ b/Frontend/library/src/Inputs/LockedMouseEvents.ts @@ -6,6 +6,7 @@ import { IMouseEvents } from './IMouseEvents'; import { NormalizedQuantizedUnsignedCoord } from '../Util/CoordinateConverter'; import { ActiveKeys } from './InputClassesFactory'; import { VideoPlayer } from '../VideoPlayer/VideoPlayer'; +import { EventListenerTracker } from '../Util/EventListenerTracker'; /** * Handle the mouse locked events @@ -21,6 +22,9 @@ export class LockedMouseEvents implements IMouseEvents { this.updateMouseMovePosition(mouseEvent); }; + // Utility for keeping track of event handlers and unregistering them + private mouseEventListenerTracker = new EventListenerTracker(); + /** * @param videoElementProvider - Video Player instance * @param mouseController - Mouse controller instance @@ -46,6 +50,13 @@ export class LockedMouseEvents implements IMouseEvents { ); } + /** + * Unregisters all event handlers + */ + unregisterMouseEvents() { + this.mouseEventListenerTracker.unregisterAll(); + } + /** * Handle when the locked state Changed */ @@ -65,6 +76,13 @@ export class LockedMouseEvents implements IMouseEvents { this.updateMouseMovePositionEvent, false ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener( + 'mousemove', + this.updateMouseMovePositionEvent, + false + ) + ); } else { Logger.Log( Logger.GetStackTrace(), diff --git a/Frontend/library/src/Inputs/MouseController.ts b/Frontend/library/src/Inputs/MouseController.ts index 14f14a2c..611d770d 100644 --- a/Frontend/library/src/Inputs/MouseController.ts +++ b/Frontend/library/src/Inputs/MouseController.ts @@ -5,6 +5,11 @@ import { Logger } from '../Logger/Logger'; import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController'; import { CoordinateConverter } from '../Util/CoordinateConverter'; import { VideoPlayer } from '../VideoPlayer/VideoPlayer'; +import { IMouseEvents } from './IMouseEvents'; +import { LockedMouseEvents } from './LockedMouseEvents'; +import { HoveringMouseEvents } from './HoveringMouseEvents'; +import type { ActiveKeys } from './InputClassesFactory'; +import { EventListenerTracker } from '../Util/EventListenerTracker'; /** * Handles the Mouse Inputs for the document @@ -13,6 +18,10 @@ export class MouseController { videoElementProvider: VideoPlayer; toStreamerMessagesProvider: StreamMessageController; coordinateConverter: CoordinateConverter; + activeKeysProvider: ActiveKeys; + + // Utility for keeping track of event handlers and unregistering them + private mouseEventListenerTracker = new EventListenerTracker(); /** * @param toStreamerMessagesProvider - Stream message instance @@ -22,26 +31,160 @@ export class MouseController { constructor( toStreamerMessagesProvider: StreamMessageController, videoElementProvider: VideoPlayer, - coordinateConverter: CoordinateConverter + coordinateConverter: CoordinateConverter, + activeKeysProvider: ActiveKeys ) { this.toStreamerMessagesProvider = toStreamerMessagesProvider; this.coordinateConverter = coordinateConverter; this.videoElementProvider = videoElementProvider; + this.activeKeysProvider = activeKeysProvider; this.registerMouseEnterAndLeaveEvents(); } /** * Clears all the click events on the current video element parent div */ - clearMouseEvents() { + unregisterMouseEvents() { + this.mouseEventListenerTracker.unregisterAll(); + } + + /** + * Register a locked mouse class + * @param mouseController - a mouse controller instance + * @param playerStyleAttributesProvider - a player style attributes instance + */ + registerLockedMouseEvents(mouseController: MouseController) { const videoElementParent = this.videoElementProvider.getVideoParentElement() as HTMLDivElement; - videoElementParent.onclick = null; - videoElementParent.onmousedown = null; - videoElementParent.onmouseup = null; - videoElementParent.onwheel = null; - videoElementParent.onmousemove = null; - videoElementParent.oncontextmenu = null; + const lockedMouseEvents: IMouseEvents = new LockedMouseEvents( + this.videoElementProvider, + mouseController, + this.activeKeysProvider + ); + + videoElementParent.requestPointerLock = + videoElementParent.requestPointerLock || + videoElementParent.mozRequestPointerLock; + document.exitPointerLock = + document.exitPointerLock || document.mozExitPointerLock; + + // minor hack to alleviate ios not supporting pointerlock + if (videoElementParent.requestPointerLock) { + const onclick = () => { + videoElementParent.requestPointerLock(); + }; + videoElementParent.addEventListener('click', onclick); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('click', onclick) + ); + } + + const lockStateChangeListener = () => + lockedMouseEvents.lockStateChange(); + document.addEventListener( + 'pointerlockchange', + lockStateChangeListener, + false + ); + document.addEventListener( + 'mozpointerlockchange', + lockStateChangeListener, + false + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener( + 'pointerlockchange', + lockStateChangeListener, + false + ) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener( + 'mozpointerlockchange', + lockStateChangeListener, + false + ) + ); + + const onmousedown = (mouseEvent: MouseEvent) => + lockedMouseEvents.handleMouseDown(mouseEvent); + const onmouseup = (mouseEvent: MouseEvent) => + lockedMouseEvents.handleMouseUp(mouseEvent); + const onwheel = (wheelEvent: WheelEvent) => + lockedMouseEvents.handleMouseWheel(wheelEvent); + const ondblclick = (mouseEvent: MouseEvent) => + lockedMouseEvents.handleMouseDouble(mouseEvent); + videoElementParent.addEventListener('mousedown', onmousedown); + videoElementParent.addEventListener('mouseup', onmouseup); + videoElementParent.addEventListener('wheel', onwheel); + videoElementParent.addEventListener('dblclick', ondblclick); + + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('mousedown', onmousedown) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('mouseup', onmouseup) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('wheel', onwheel) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('dblclick', ondblclick) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => lockedMouseEvents.unregisterMouseEvents() + ); + } + + /** + * Register a hovering mouse class + * @param mouseController - A mouse controller object + */ + registerHoveringMouseEvents(mouseController: MouseController) { + const videoElementParent = + this.videoElementProvider.getVideoParentElement() as HTMLDivElement; + const hoveringMouseEvents = new HoveringMouseEvents(mouseController); + + const onmousemove = (mouseEvent: MouseEvent) => + hoveringMouseEvents.updateMouseMovePosition(mouseEvent); + const onmousedown = (mouseEvent: MouseEvent) => + hoveringMouseEvents.handleMouseDown(mouseEvent); + const onmouseup = (mouseEvent: MouseEvent) => + hoveringMouseEvents.handleMouseUp(mouseEvent); + const oncontextmenu = (mouseEvent: MouseEvent) => + hoveringMouseEvents.handleContextMenu(mouseEvent); + const onwheel = (wheelEvent: WheelEvent) => + hoveringMouseEvents.handleMouseWheel(wheelEvent); + const ondblclick = (mouseEvent: MouseEvent) => + hoveringMouseEvents.handleMouseDouble(mouseEvent); + videoElementParent.addEventListener('mousemove', onmousemove); + videoElementParent.addEventListener('mousedown', onmousedown); + videoElementParent.addEventListener('mouseup', onmouseup); + videoElementParent.addEventListener('contextmenu', oncontextmenu); + videoElementParent.addEventListener('wheel', onwheel); + videoElementParent.addEventListener('dblclick', ondblclick); + + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('mousemove', onmousemove) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('mousedown', onmousedown) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('mouseup', onmouseup) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('contextmenu', oncontextmenu) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('wheel', onwheel) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('dblclick', ondblclick) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => hoveringMouseEvents.unregisterMouseEvents() + ); } /** @@ -52,7 +195,7 @@ export class MouseController { this.videoElementProvider.getVideoParentElement() as HTMLDivElement; // Handle when the Mouse has entered the element - videoElementParent.onmouseenter = (event: MouseEvent) => { + const onmouseenter = (event: MouseEvent) => { if (!this.videoElementProvider.isVideoReady()) { return; } @@ -62,7 +205,7 @@ export class MouseController { }; // Handles when the mouse has left the element - videoElementParent.onmouseleave = (event: MouseEvent) => { + const onmouseleave = (event: MouseEvent) => { if (!this.videoElementProvider.isVideoReady()) { return; } @@ -70,6 +213,15 @@ export class MouseController { this.sendMouseLeave(); this.releaseMouseButtons(event.buttons, event.x, event.y); }; + videoElementParent.addEventListener('mouseenter', onmouseenter); + videoElementParent.addEventListener('mouseleave', onmouseleave); + + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('mouseenter', onmouseenter) + ); + this.mouseEventListenerTracker.addUnregisterCallback( + () => videoElementParent.removeEventListener('mouseleave', onmouseleave) + ); } /** diff --git a/Frontend/library/src/Inputs/TouchController.ts b/Frontend/library/src/Inputs/TouchController.ts index ff8209cd..83aec684 100644 --- a/Frontend/library/src/Inputs/TouchController.ts +++ b/Frontend/library/src/Inputs/TouchController.ts @@ -5,6 +5,7 @@ import { CoordinateConverter } from '../Util/CoordinateConverter'; import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController'; import { VideoPlayer } from '../VideoPlayer/VideoPlayer'; import { ITouchController } from './ITouchController'; +import { EventListenerTracker } from '../Util/EventListenerTracker'; /** * Handles the Touch input Events */ @@ -17,6 +18,9 @@ export class TouchController implements ITouchController { fingerIds = new Map(); maxByteValue = 255; + // Utility for keeping track of event handlers and unregistering them + private touchEventListenerTracker = new EventListenerTracker(); + /** * @param toStreamerMessagesProvider - Stream message instance * @param videoElementProvider - Video Player instance @@ -31,18 +35,41 @@ export class TouchController implements ITouchController { this.videoElementProvider = videoElementProvider; this.coordinateConverter = coordinateConverter; this.videoElementParent = videoElementProvider.getVideoElement(); - this.videoElementParent.ontouchstart = (ev: TouchEvent) => + const ontouchstart = (ev: TouchEvent) => this.onTouchStart(ev); - this.videoElementParent.ontouchend = (ev: TouchEvent) => + const ontouchend = (ev: TouchEvent) => this.onTouchEnd(ev); - this.videoElementParent.ontouchmove = (ev: TouchEvent) => + const ontouchmove = (ev: TouchEvent) => this.onTouchMove(ev); + this.videoElementParent.addEventListener('touchstart', ontouchstart, { passive: false }); + this.videoElementParent.addEventListener('touchend', ontouchend, { passive: false }); + this.videoElementParent.addEventListener('touchmove', ontouchmove, { passive: false }); + this.touchEventListenerTracker.addUnregisterCallback( + () => this.videoElementParent.removeEventListener('touchstart', ontouchstart) + ); + this.touchEventListenerTracker.addUnregisterCallback( + () => this.videoElementParent.removeEventListener('touchend', ontouchend) + ); + this.touchEventListenerTracker.addUnregisterCallback( + () => this.videoElementParent.removeEventListener('touchmove', ontouchmove) + ); Logger.Log(Logger.GetStackTrace(), 'Touch Events Registered', 6); // is this strictly necessary? - document.ontouchmove = (event: TouchEvent) => { + const preventOnTouchMove = (event: TouchEvent) => { event.preventDefault(); }; + document.addEventListener('touchmove', preventOnTouchMove, { passive: false }); + this.touchEventListenerTracker.addUnregisterCallback( + () => document.removeEventListener('touchmove', preventOnTouchMove) + ); + } + + /** + * Unregister all touch events + */ + unregisterTouchEvents() { + this.touchEventListenerTracker.unregisterAll(); } /** diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index ee8bccad..eaa9da07 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -153,7 +153,36 @@ export class PixelStreaming { isHoveringMouse ? 'Hovering' : 'Locked' } Mouse` ); - this._webRtcController.activateRegisterMouse(); + this._webRtcController.setMouseInputEnabled(this.config.isFlagEnabled(Flags.MouseInput)); + } + ); + + // user input + this.config._addOnSettingChangedListener( + Flags.KeyboardInput, + (isEnabled: boolean) => { + this._webRtcController.setKeyboardInputEnabled(isEnabled); + } + ); + + this.config._addOnSettingChangedListener( + Flags.MouseInput, + (isEnabled: boolean) => { + this._webRtcController.setMouseInputEnabled(isEnabled); + } + ); + + this.config._addOnSettingChangedListener( + Flags.TouchInput, + (isEnabled: boolean) => { + this._webRtcController.setTouchInputEnabled(isEnabled); + } + ); + + this.config._addOnSettingChangedListener( + Flags.GamepadInput, + (isEnabled: boolean) => { + this._webRtcController.setGamePadInputEnabled(isEnabled); } ); diff --git a/Frontend/library/src/Util/EventListenerTracker.ts b/Frontend/library/src/Util/EventListenerTracker.ts new file mode 100644 index 00000000..c0058690 --- /dev/null +++ b/Frontend/library/src/Util/EventListenerTracker.ts @@ -0,0 +1,29 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +export type UnregisterFunction = () => void; + +export class EventListenerTracker { + private unregisterCallbacks: UnregisterFunction[]; + + constructor() { + this.unregisterCallbacks = []; + } + + /** + * Add a new callback that is executed when unregisterAll is called. + * @param callback + */ + addUnregisterCallback(callback: UnregisterFunction) { + this.unregisterCallbacks.push(callback); + } + + /** + * Execute all callbacks and clear the list. + */ + unregisterAll() { + for (const callback of this.unregisterCallbacks) { + callback(); + } + this.unregisterCallbacks = []; + } +} diff --git a/Frontend/library/src/VideoPlayer/VideoPlayer.ts b/Frontend/library/src/VideoPlayer/VideoPlayer.ts index 0cf4d8e3..2e432174 100644 --- a/Frontend/library/src/VideoPlayer/VideoPlayer.ts +++ b/Frontend/library/src/VideoPlayer/VideoPlayer.ts @@ -8,8 +8,6 @@ import { Logger } from '../Logger/Logger'; */ declare global { interface HTMLElement { - pressMouseButtons?(mouseEvent: MouseEvent): void; - releaseMouseButtons?(mouseEvent: MouseEvent): void; mozRequestPointerLock?(): void; } } diff --git a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts index 6d4e0b49..1d13e2c1 100644 --- a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts +++ b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts @@ -1026,10 +1026,7 @@ export class WebRtcPlayerController { return; } - this.touchController = this.inputClassesFactory.registerTouch( - this.config.isFlagEnabled(Flags.FakeMouseWithTouches), - this.videoElementParentClientRect - ); + this.setTouchInputEnabled(this.config.isFlagEnabled(Flags.TouchInput)); this.pixelStreaming.dispatchEvent(new PlayStreamEvent()); if (this.streamController.audioElement.srcObject) { @@ -1421,11 +1418,9 @@ export class WebRtcPlayerController { this.statsTimerHandle = window.setInterval(() => this.getStats(), 1000); /* */ - this.activateRegisterMouse(); - this.keyboardController = this.inputClassesFactory.registerKeyBoard( - this.config - ); - this.gamePadController = this.inputClassesFactory.registerGamePad(); + this.setMouseInputEnabled(this.config.isFlagEnabled(Flags.MouseInput)); + this.setKeyboardInputEnabled(this.config.isFlagEnabled(Flags.KeyboardInput)); + this.setGamePadInputEnabled(this.config.isFlagEnabled(Flags.GamepadInput)); } /** @@ -1503,17 +1498,6 @@ export class WebRtcPlayerController { } } - /** - * registers the mouse for use in WebRtcPlayerController - */ - activateRegisterMouse() { - const mouseMode = this.config.isFlagEnabled(Flags.HoveringMouseMode) - ? ControlSchemeType.HoveringMouse - : ControlSchemeType.LockedMouse; - this.mouseController = - this.inputClassesFactory.registerMouse(mouseMode); - } - /** * Set the freeze frame overlay to the player div */ @@ -1900,6 +1884,55 @@ export class WebRtcPlayerController { this.pixelStreaming._onVideoEncoderAvgQP(this.videoAvgQp); } + /** + * enables/disables keyboard event listeners + */ + setKeyboardInputEnabled(isEnabled: boolean) { + this.keyboardController?.unregisterKeyBoardEvents(); + if (isEnabled) { + this.keyboardController = this.inputClassesFactory.registerKeyBoard( + this.config + ); + } + } + + /** + * enables/disables mouse event listeners + */ + setMouseInputEnabled(isEnabled: boolean) { + this.mouseController?.unregisterMouseEvents(); + if (isEnabled) { + const mouseMode = this.config.isFlagEnabled(Flags.HoveringMouseMode) + ? ControlSchemeType.HoveringMouse + : ControlSchemeType.LockedMouse; + this.mouseController = + this.inputClassesFactory.registerMouse(mouseMode); + } + } + + /** + * enables/disables touch event listeners + */ + setTouchInputEnabled(isEnabled: boolean) { + this.touchController?.unregisterTouchEvents(); + if (isEnabled) { + this.touchController = this.inputClassesFactory.registerTouch( + this.config.isFlagEnabled(Flags.FakeMouseWithTouches), + this.videoElementParentClientRect + ); + } + } + + /** + * enables/disables game pad event listeners + */ + setGamePadInputEnabled(isEnabled: boolean) { + this.gamePadController?.unregisterGamePadEvents(); + if (isEnabled) { + this.gamePadController = this.inputClassesFactory.registerGamePad(); + } + } + registerDataChannelEventEmitters(dataChannel: DataChannelController) { dataChannel.onOpen = (label, event) => this.pixelStreaming.dispatchEvent( diff --git a/Frontend/library/src/WebXR/WebXRController.ts b/Frontend/library/src/WebXR/WebXRController.ts index c2d087b6..0f6ae948 100644 --- a/Frontend/library/src/WebXR/WebXRController.ts +++ b/Frontend/library/src/WebXR/WebXRController.ts @@ -6,6 +6,7 @@ import { WebGLUtils } from '../Util/WebGLUtils'; import { Controller } from '../Inputs/GamepadTypes'; import { XRGamepadController } from '../Inputs/XRGamepadController'; import { XrFrameEvent } from '../Util/EventEmitter' +import { Flags } from '../pixelstreamingfrontend'; export class WebXRController { private xrSession: XRSession; @@ -190,16 +191,18 @@ export class WebXRController { this.render(this.webRtcController.videoPlayer.getVideoElement()); } - this.xrSession.inputSources.forEach( - (source: XRInputSource, index: number, array: XRInputSource[]) => { - this.xrGamepadController.updateStatus( - source, - frame, - this.xrRefSpace - ); - }, - this - ); + if (this.webRtcController.config.isFlagEnabled(Flags.XRControllerInput)) { + this.xrSession.inputSources.forEach( + (source: XRInputSource, index: number, array: XRInputSource[]) => { + this.xrGamepadController.updateStatus( + source, + frame, + this.xrRefSpace + ); + }, + this + ); + } this.xrSession.requestAnimationFrame( (time: DOMHighResTimeStamp, frame: XRFrame) => diff --git a/Frontend/ui-library/package-lock.json b/Frontend/ui-library/package-lock.json index b36eee45..332fbd66 100644 --- a/Frontend/ui-library/package-lock.json +++ b/Frontend/ui-library/package-lock.json @@ -1,12 +1,12 @@ { "name": "@epicgames-ps/lib-pixelstreamingfrontend-ui-ue5.2", - "version": "0.0.3", + "version": "0.0.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@epicgames-ps/lib-pixelstreamingfrontend-ui-ue5.2", - "version": "0.0.3", + "version": "0.0.4", "license": "MIT", "dependencies": { "jss": "^10.9.2", diff --git a/Frontend/ui-library/src/Config/ConfigUI.ts b/Frontend/ui-library/src/Config/ConfigUI.ts index 43891f22..ffcb0665 100644 --- a/Frontend/ui-library/src/Config/ConfigUI.ts +++ b/Frontend/ui-library/src/Config/ConfigUI.ts @@ -220,6 +220,37 @@ export class ConfigUI { this.addSettingFlag(viewSettingsSection, this.flagsUi.get(LightMode)); + /* Setup all encoder related settings under this section */ + const inputSettingsSection = this.buildSectionWithHeading( + settingsElem, + 'Input' + ); + + this.addSettingFlag( + inputSettingsSection, + this.flagsUi.get(Flags.KeyboardInput) + ); + + this.addSettingFlag( + inputSettingsSection, + this.flagsUi.get(Flags.MouseInput) + ); + + this.addSettingFlag( + inputSettingsSection, + this.flagsUi.get(Flags.TouchInput) + ); + + this.addSettingFlag( + inputSettingsSection, + this.flagsUi.get(Flags.GamepadInput) + ); + + this.addSettingFlag( + inputSettingsSection, + this.flagsUi.get(Flags.XRControllerInput) + ); + /* Setup all encoder related settings under this section */ const encoderSettingsSection = this.buildSectionWithHeading( settingsElem,