diff --git a/Frontend/library/package-lock.json b/Frontend/library/package-lock.json index d7756ab5..dbd1a6a4 100644 --- a/Frontend/library/package-lock.json +++ b/Frontend/library/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "license": "MIT", "dependencies": { + "@types/webxr": "^0.5.1", "jss": "^10.9.2", "jss-plugin-camel-case": "^10.9.2", "jss-plugin-global": "^10.9.2", @@ -438,6 +439,11 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, + "node_modules/@types/webxr": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.1.tgz", + "integrity": "sha512-xlFXPfgJR5vIuDefhaHuUM9uUgvPaXB6GKdXy2gdEh8gBWQZ2ul24AJz3foUd8NNKlSTQuWYJpCb1/pL81m1KQ==" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.48.0.tgz", @@ -3952,6 +3958,11 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, + "@types/webxr": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.1.tgz", + "integrity": "sha512-xlFXPfgJR5vIuDefhaHuUM9uUgvPaXB6GKdXy2gdEh8gBWQZ2ul24AJz3foUd8NNKlSTQuWYJpCb1/pL81m1KQ==" + }, "@typescript-eslint/eslint-plugin": { "version": "5.48.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.48.0.tgz", diff --git a/Frontend/library/package.json b/Frontend/library/package.json index f89bc63f..707374dc 100644 --- a/Frontend/library/package.json +++ b/Frontend/library/package.json @@ -24,6 +24,7 @@ "webpack-cli": "^5.0.1" }, "dependencies": { + "@types/webxr": "^0.5.1", "jss": "^10.9.2", "jss-plugin-camel-case": "^10.9.2", "jss-plugin-global": "^10.9.2", diff --git a/Frontend/library/src/Application/Application.ts b/Frontend/library/src/Application/Application.ts index 1a04b11c..be915d39 100644 --- a/Frontend/library/src/Application/Application.ts +++ b/Frontend/library/src/Application/Application.ts @@ -26,12 +26,14 @@ import { PlayOverlay } from '../Overlay/PlayOverlay'; import { InfoOverlay } from '../Overlay/InfoOverlay'; import { ErrorOverlay } from '../Overlay/ErrorOverlay'; import { MessageOnScreenKeyboard } from '../WebSockets/MessageReceive'; +import { WebXRController } from '../WebXR/WebXRController'; /** * Provides common base functionality for applications that extend this application */ export class Application { public webRtcController: WebRtcPlayerController; + public webXrController: WebXRController; public config: Config; _rootElement: HTMLElement; @@ -101,6 +103,8 @@ export class Application { this.onScreenKeyboardHelper.showOnScreenKeyboard(command); this.updateColors(this.config.isFlagEnabled(Flags.LightMode)); + + this.webXrController = new WebXRController(this.webRtcController); } public createOverlays(): void { @@ -129,6 +133,10 @@ export class Application { this.settingsPanel.settingsCloseButton.onclick = () => this.settingsClicked(); + // Add WebXR button to controls + controls.xrIcon.rootElement.onclick = () => + this.webXrController.xrClicked(); + // setup the stats/info button controls.statsIcon.rootElement.onclick = () => this.statsClicked(); @@ -155,7 +163,7 @@ export class Application { 'Request' ); requestKeyframeButton.addOnClickListener(() => { - this.webRtcController.requestKeyFrame(); + this.webRtcController.sendIframeRequest(); }); const commandsSectionElem = this.config.buildSectionWithHeading( @@ -394,14 +402,14 @@ export class Application { } ); - this.config.addOnOptionSettingChangedListener( - OptionParameters.PreferredCodec, - (newValue: string) => { - if(this.webRtcController) { - this.webRtcController.setPreferredCodec(newValue); - } - } - ); + this.config.addOnOptionSettingChangedListener( + OptionParameters.PreferredCodec, + (newValue: string) => { + if (this.webRtcController) { + this.webRtcController.setPreferredCodec(newValue); + } + } + ); } /** @@ -547,7 +555,10 @@ export class Application { setWebRtcPlayerController(webRtcPlayerController: WebRtcPlayerController) { this.webRtcController = webRtcPlayerController; - this.webRtcController.setPreferredCodec(this.config.getSettingOption(OptionParameters.PreferredCodec).selected); + this.webRtcController.setPreferredCodec( + this.config.getSettingOption(OptionParameters.PreferredCodec) + .selected + ); this.webRtcController.resizePlayerStyle(); this.disconnectOverlay.onAction(() => { diff --git a/Frontend/library/src/Config/Config.ts b/Frontend/library/src/Config/Config.ts index 9bb64ea2..28268729 100644 --- a/Frontend/library/src/Config/Config.ts +++ b/Frontend/library/src/Config/Config.ts @@ -55,7 +55,7 @@ export class TextParameters { * */ export class OptionParameters { - static PreferredCodec = 'PreferredCodec'; + static PreferredCodec = 'PreferredCodec'; static StreamerId = 'StreamerId'; } @@ -69,8 +69,8 @@ export class Config { /* A map of text settings - e.g. signalling server url */ private textParameters = new Map(); - /* A map of enum based settings - e.g. preferred codec */ - private optionParameters = new Map(); + /* A map of enum based settings - e.g. preferred codec */ + private optionParameters = new Map(); // ------------ Settings ----------------- @@ -116,52 +116,61 @@ export class Config { 'Signalling url', 'Url of the signalling server', (location.protocol === 'https:' ? 'wss://' : 'ws://') + - window.location.hostname + - // for readability, we omit the port if it's 80 - ((window.location.port === '80' || window.location.port === '') ? '' : `:${window.location.port}`) + window.location.hostname + + // for readability, we omit the port if it's 80 + (window.location.port === '80' || + window.location.port === '' + ? '' + : `:${window.location.port}`) ) ); - this.optionParameters.set(OptionParameters.StreamerId, + this.optionParameters.set( + OptionParameters.StreamerId, new SettingOption( OptionParameters.StreamerId, 'Streamer ID', 'The ID of the streamer to stream.', '', - []) - ); - - /** - * Enum Parameters - */ - this.optionParameters.set(OptionParameters.PreferredCodec, - new SettingOption( - OptionParameters.PreferredCodec, - 'Preferred Codec', - 'The preferred codec to be used during codec negotiation', - 'H264 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f', - (function(): Array { - const browserSupportedCodecs: Array = []; - // Try get the info needed from the RTCRtpReceiver. This is only available on chrome - if(!RTCRtpReceiver.getCapabilities) - { - browserSupportedCodecs.push("Only available on Chrome"); - return browserSupportedCodecs; - } - - const matcher = /(VP\d|H26\d|AV1).*/ - const codecs = RTCRtpReceiver.getCapabilities('video').codecs; - codecs.forEach(codec => { - const str = codec.mimeType.split("/")[1] + ' ' + (codec.sdpFmtpLine || ''); - const match = matcher.exec(str); - if(match !== null) { - browserSupportedCodecs.push(str); - } - }); - return browserSupportedCodecs; - })() - ) - ); + [] + ) + ); + + /** + * Enum Parameters + */ + this.optionParameters.set( + OptionParameters.PreferredCodec, + new SettingOption( + OptionParameters.PreferredCodec, + 'Preferred Codec', + 'The preferred codec to be used during codec negotiation', + 'H264 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f', + (function (): Array { + const browserSupportedCodecs: Array = []; + // Try get the info needed from the RTCRtpReceiver. This is only available on chrome + if (!RTCRtpReceiver.getCapabilities) { + browserSupportedCodecs.push('Only available on Chrome'); + return browserSupportedCodecs; + } + + const matcher = /(VP\d|H26\d|AV1).*/; + const codecs = + RTCRtpReceiver.getCapabilities('video').codecs; + codecs.forEach((codec) => { + const str = + codec.mimeType.split('/')[1] + + ' ' + + (codec.sdpFmtpLine || ''); + const match = matcher.exec(str); + if (match !== null) { + browserSupportedCodecs.push(str); + } + }); + return browserSupportedCodecs; + })() + ) + ); /** * Boolean parameters @@ -454,14 +463,17 @@ export class Config { this.numericParameters.get(NumericParameters.AFKTimeoutSecs) ); - const preferredCodecOption = this.optionParameters.get(OptionParameters.PreferredCodec); - this.addSettingOption( - psSettingsSection, - preferredCodecOption - ); - if([...preferredCodecOption.selector.options].map(o => o.value).includes("Only available on Chrome")) { - preferredCodecOption.disable(); - } + const preferredCodecOption = this.optionParameters.get( + OptionParameters.PreferredCodec + ); + this.addSettingOption(psSettingsSection, preferredCodecOption); + if ( + [...preferredCodecOption.selector.options] + .map((o) => o.value) + .includes('Only available on Chrome') + ) { + preferredCodecOption.disable(); + } /* Setup all view/ui related settings under this section */ const viewSettingsSection = this.buildSectionWithHeading( @@ -536,14 +548,16 @@ export class Config { } } - addOnOptionSettingChangedListener( - id: string, + addOnOptionSettingChangedListener( + id: string, onChangedListener: (newValue: string) => void - ): void { - if(this.optionParameters.has(id)) { - this.optionParameters.get(id).addOnChangedListener(onChangedListener); - } - } + ): void { + if (this.optionParameters.has(id)) { + this.optionParameters + .get(id) + .addOnChangedListener(onChangedListener); + } + } /** * @param id The id of the numeric setting we are interested in getting a value for. @@ -649,21 +663,22 @@ export class Config { this.numericParameters.set(setting.id, setting); } - /** + /** * Add an enum based settings element to a particular settings section in the DOM and registers that flag in the Config.enumParameters map. * @param settingsSection The settings section HTML element. * @param settingFlag The settings flag object. */ - addSettingOption(settingsSection: HTMLElement, - setting: SettingOption - ): void { - settingsSection.appendChild(setting.rootElement); - this.optionParameters.set(setting.id, setting); - } + addSettingOption( + settingsSection: HTMLElement, + setting: SettingOption + ): void { + settingsSection.appendChild(setting.rootElement); + this.optionParameters.set(setting.id, setting); + } - getSettingOption(id: string): SettingOption { - return this.optionParameters.get(id); - } + getSettingOption(id: string): SettingOption { + return this.optionParameters.get(id); + } /** * Get the value of the configuration flag which has the given id. @@ -706,7 +721,7 @@ export class Config { } } - /** + /** * Set the option setting list of options. * @param id The id of the setting * @param settingOptions The values the setting could take @@ -722,7 +737,7 @@ export class Config { } } - /** + /** * Set option enum settings selected option. * @param id The id of the setting * @param settingOptions The value to select out of all the options diff --git a/Frontend/library/src/Config/SettingOption.ts b/Frontend/library/src/Config/SettingOption.ts index b9feff76..88062cdb 100644 --- a/Frontend/library/src/Config/SettingOption.ts +++ b/Frontend/library/src/Config/SettingOption.ts @@ -3,29 +3,31 @@ import { SettingBase } from './SettingBase'; export class SettingOption extends SettingBase { - /* A select element that reflects the value of this setting. */ + /* A select element that reflects the value of this setting. */ _selector: HTMLSelectElement; // /* This element contains a text node that reflects the setting's text label. */ _settingsTextElem: HTMLElement; - constructor( + constructor( id: string, label: string, description: string, defaultTextValue: string, - options: Array + options: Array ) { super(id, label, description, [defaultTextValue, defaultTextValue]); - this.options = options + this.options = options; const urlParams = new URLSearchParams(window.location.search); - const stringToMatch: string = (urlParams.has(this.id)) ? this.getUrlParamText() : defaultTextValue; - this.selected = stringToMatch; + const stringToMatch: string = urlParams.has(this.id) + ? this.getUrlParamText() + : defaultTextValue; + this.selected = stringToMatch; } - public get selector(): HTMLSelectElement { + public get selector(): HTMLSelectElement { if (!this._selector) { this._selector = document.createElement('select'); this._selector.classList.add('form-control'); @@ -33,7 +35,7 @@ export class SettingOption extends SettingBase { return this._selector; } - public get settingsTextElem(): HTMLElement { + public get settingsTextElem(): HTMLElement { if (!this._settingsTextElem) { this._settingsTextElem = document.createElement('div'); this._settingsTextElem.innerText = this._label; @@ -51,7 +53,7 @@ export class SettingOption extends SettingBase { this.settingsTextElem.innerText = this._label; } - /** + /** * @returns Return or creates a HTML element that represents this setting in the DOM. */ public get rootElement(): HTMLElement { @@ -74,7 +76,7 @@ export class SettingOption extends SettingBase { // setup on change from selector this.selector.onchange = () => { - this.value = this.selector.value; + this.value = this.selector.value; // set url params const urlParams = new URLSearchParams(window.location.search); @@ -91,7 +93,7 @@ export class SettingOption extends SettingBase { return this._rootElement; } - /** + /** * Parse the text value from the url parameters. * @returns The text value parsed from the url if the url parameters contains /?id=value, but empty string if just /?id or no url param found. */ @@ -103,49 +105,51 @@ export class SettingOption extends SettingBase { return ''; } - /** + /** * Add a change listener to the select element. */ public addOnChangedListener(onChangedFunc: (newValue: string) => void) { this.onChange = onChangedFunc; } - public get options(): Array { - return [...this.selector.options].map(o => o.value); - } - - public set options(values: Array) { - for (let i = this.selector.options.length - 1; i >= 0; i--) { - this.selector.remove(i); - } - - values.forEach((value: string) => { - const opt = document.createElement('option'); - opt.value = value; - opt.innerHTML = value; - this.selector.appendChild(opt); - }); - } - - public get selected(): string { - return this.value as string; - } - - public set selected(value: string) { - // A user may not specify the full possible value so we instead use the closest match. - // eg ?xxx=H264 would select 'H264 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f' - const filteredList = this.options.filter((option: string) => option.indexOf(value) !== -1); - if(filteredList.length) { - this.value = filteredList[0]; - this.selector.value = filteredList[0]; - } - } - - public disable() { - this.selector.disabled = true; - } - - public enable() { - this.selector.disabled = false; - } -} \ No newline at end of file + public get options(): Array { + return [...this.selector.options].map((o) => o.value); + } + + public set options(values: Array) { + for (let i = this.selector.options.length - 1; i >= 0; i--) { + this.selector.remove(i); + } + + values.forEach((value: string) => { + const opt = document.createElement('option'); + opt.value = value; + opt.innerHTML = value; + this.selector.appendChild(opt); + }); + } + + public get selected(): string { + return this.value as string; + } + + public set selected(value: string) { + // A user may not specify the full possible value so we instead use the closest match. + // eg ?xxx=H264 would select 'H264 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f' + const filteredList = this.options.filter( + (option: string) => option.indexOf(value) !== -1 + ); + if (filteredList.length) { + this.value = filteredList[0]; + this.selector.value = filteredList[0]; + } + } + + public disable() { + this.selector.disabled = true; + } + + public enable() { + this.selector.disabled = false; + } +} diff --git a/Frontend/library/src/Inputs/FakeTouchController.ts b/Frontend/library/src/Inputs/FakeTouchController.ts index f45144f2..982a2f2d 100644 --- a/Frontend/library/src/Inputs/FakeTouchController.ts +++ b/Frontend/library/src/Inputs/FakeTouchController.ts @@ -70,8 +70,8 @@ export class FakeTouchController implements ITouchController { this.fakeTouchFinger.y ); const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseDown')('MouseDown', [ + this.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseDown')([ MouseButton.mainButton, coord.x, coord.y @@ -91,7 +91,7 @@ export class FakeTouchController implements ITouchController { const videoElementParent = this.videoElementProvider.getVideoParentElement(); const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); + this.toStreamerMessagesProvider.toStreamerHandlers; for (let t = 0; t < touchEvent.changedTouches.length; t++) { const touch = touchEvent.changedTouches[t]; @@ -101,7 +101,7 @@ export class FakeTouchController implements ITouchController { const y = touch.clientY - this.videoElementParentClientRect.top; const coord = this.coordinateConverter.normalizeAndQuantizeUnsigned(x, y); - toStreamerHandlers.get('MouseUp')('MouseUp', [ + toStreamerHandlers.get('MouseUp')([ MouseButton.mainButton, coord.x, coord.y @@ -125,7 +125,7 @@ export class FakeTouchController implements ITouchController { return; } const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); + this.toStreamerMessagesProvider.toStreamerHandlers; for (let t = 0; t < touchEvent.touches.length; t++) { const touch = touchEvent.touches[t]; @@ -140,7 +140,7 @@ export class FakeTouchController implements ITouchController { x - this.fakeTouchFinger.x, y - this.fakeTouchFinger.y ); - toStreamerHandlers.get('MoveMouse')('MouseMove', [ + toStreamerHandlers.get('MoveMouse')([ coord.x, coord.y, delta.x, diff --git a/Frontend/library/src/Inputs/GamepadController.ts b/Frontend/library/src/Inputs/GamepadController.ts index 8d7448a2..cdba4bbb 100644 --- a/Frontend/library/src/Inputs/GamepadController.ts +++ b/Frontend/library/src/Inputs/GamepadController.ts @@ -2,12 +2,13 @@ import { Logger } from '../Logger/Logger'; import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController'; +import { Controller } from './GamepadTypes'; /** * The class that handles the functionality of gamepads and controllers */ export class GamePadController { - controllers: Controller[]; + controllers: Array; requestAnimationFrame: (callback: FrameRequestCallback) => number; toStreamerMessagesProvider: StreamMessageController; @@ -105,7 +106,7 @@ export class GamePadController { updateStatus() { this.scanGamePads(); const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); + this.toStreamerMessagesProvider.toStreamerHandlers; // Iterate over multiple controllers in the case the multiple gamepads are connected for (const controller of this.controllers) { @@ -118,41 +119,46 @@ export class GamePadController { // press if (i == gamepadLayout.LeftTrigger) { // UEs left analog has a button index of 5 - toStreamerHandlers.get('GamepadAnalog')( - 'GamepadAnalog', - [controllerIndex, 5, currentButton.value] - ); + toStreamerHandlers.get('GamepadAnalog')([ + controllerIndex, + 5, + currentButton.value + ]); } else if (i == gamepadLayout.RightTrigger) { // UEs right analog has a button index of 6 - toStreamerHandlers.get('GamepadAnalog')( - 'GamepadAnalog', - [controllerIndex, 6, currentButton.value] - ); + toStreamerHandlers.get('GamepadAnalog')([ + controllerIndex, + 6, + currentButton.value + ]); } else { - toStreamerHandlers.get('GamepadButtonPressed')( - 'GamepadButtonPressed', - [controllerIndex, i, previousButton.pressed ? 1 : 0] - ); + toStreamerHandlers.get('GamepadButtonPressed')([ + controllerIndex, + i, + previousButton.pressed ? 1 : 0 + ]); } } else if (!currentButton.pressed && previousButton.pressed) { // release if (i == gamepadLayout.LeftTrigger) { // UEs left analog has a button index of 5 - toStreamerHandlers.get('GamepadAnalog')( - 'GamepadAnalog', - [controllerIndex, 5, 0] - ); + toStreamerHandlers.get('GamepadAnalog')([ + controllerIndex, + 5, + 0 + ]); } else if (i == gamepadLayout.RightTrigger) { // UEs right analog has a button index of 6 - toStreamerHandlers.get('GamepadAnalog')( - 'GamepadAnalog', - [controllerIndex, 6, 0] - ); + toStreamerHandlers.get('GamepadAnalog')([ + controllerIndex, + 6, + 0 + ]); } else { - toStreamerHandlers.get('GamepadButtonReleased')( - 'GamepadButtonReleased', - [controllerIndex, i] - ); + toStreamerHandlers.get('GamepadButtonReleased')([ + controllerIndex, + i + ]); } } } @@ -166,12 +172,12 @@ export class GamePadController { const y = -parseFloat(currentState.axes[i + 1].toFixed(4)); // UE's analog axes follow the same order as the browsers, but start at index 1 so we will offset as such - toStreamerHandlers.get('GamepadAnalog')('GamepadAnalog', [ + toStreamerHandlers.get('GamepadAnalog')([ controllerIndex, i + 1, x ]); // Horizontal axes, only offset by 1 - toStreamerHandlers.get('GamepadAnalog')('GamepadAnalog', [ + toStreamerHandlers.get('GamepadAnalog')([ controllerIndex, i + 2, y @@ -224,11 +230,3 @@ export enum gamepadLayout { RightStickHorizontal = 2, RightStickVertical = 3 } - -/** - * The interface for controllers - */ -export interface Controller { - currentState: Gamepad; - prevState: Gamepad; -} diff --git a/Frontend/library/src/Inputs/GamepadTypes.ts b/Frontend/library/src/Inputs/GamepadTypes.ts new file mode 100644 index 00000000..f51fe3ee --- /dev/null +++ b/Frontend/library/src/Inputs/GamepadTypes.ts @@ -0,0 +1,9 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +/** + * The interface for controllers + */ +export interface Controller { + currentState: Gamepad; + prevState: Gamepad; +} diff --git a/Frontend/library/src/Inputs/HoveringMouseEvents.ts b/Frontend/library/src/Inputs/HoveringMouseEvents.ts index aabd7ba5..674eafa5 100644 --- a/Frontend/library/src/Inputs/HoveringMouseEvents.ts +++ b/Frontend/library/src/Inputs/HoveringMouseEvents.ts @@ -37,8 +37,8 @@ export class HoveringMouseEvents implements IMouseEvents { mouseEvent.movementY ); const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseMove')('MouseMove', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseMove')([ coord.x, coord.y, delta.x, @@ -62,8 +62,8 @@ export class HoveringMouseEvents implements IMouseEvents { mouseEvent.offsetY ); const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseDown')('MouseDown', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseDown')([ mouseEvent.button, coord.x, coord.y @@ -85,8 +85,8 @@ export class HoveringMouseEvents implements IMouseEvents { mouseEvent.offsetY ); const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseUp')('MouseUp', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseUp')([ mouseEvent.button, coord.x, coord.y @@ -108,8 +108,8 @@ export class HoveringMouseEvents implements IMouseEvents { mouseEvent.offsetY ); const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseUp')('MouseUp', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseUp')([ mouseEvent.button, coord.x, coord.y @@ -131,8 +131,8 @@ export class HoveringMouseEvents implements IMouseEvents { wheelEvent.offsetY ); const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseWheel')('MouseWheel', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseWheel')([ wheelEvent.wheelDelta, coord.x, coord.y @@ -154,8 +154,8 @@ export class HoveringMouseEvents implements IMouseEvents { mouseEvent.offsetY ); const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseDouble')('MouseDouble', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseDouble')([ mouseEvent.button, coord.x, coord.y diff --git a/Frontend/library/src/Inputs/KeyboardController.ts b/Frontend/library/src/Inputs/KeyboardController.ts index 13ea1e5d..6866c045 100644 --- a/Frontend/library/src/Inputs/KeyboardController.ts +++ b/Frontend/library/src/Inputs/KeyboardController.ts @@ -169,8 +169,8 @@ export class KeyboardController { 6 ); const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('KeyDown')('KeyDown', [ + this.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('KeyDown')([ this.getKeycode(keyboardEvent), keyboardEvent.repeat ? 1 : 0 ]); @@ -206,8 +206,8 @@ export class KeyboardController { Logger.Log(Logger.GetStackTrace(), `key up ${keyCode}`, 6); const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('KeyUp')('KeyUp', [ + this.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('KeyUp')([ keyCode, keyboardEvent.repeat ? 1 : 0 ]); @@ -237,8 +237,8 @@ export class KeyboardController { Logger.Log(Logger.GetStackTrace(), `key press ${charCode}`, 6); const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('KeyPress')('KeyPress', [charCode]); + this.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('KeyPress')([charCode]); } /** diff --git a/Frontend/library/src/Inputs/LockedMouseEvents.ts b/Frontend/library/src/Inputs/LockedMouseEvents.ts index a385675d..38e3af6b 100644 --- a/Frontend/library/src/Inputs/LockedMouseEvents.ts +++ b/Frontend/library/src/Inputs/LockedMouseEvents.ts @@ -53,7 +53,7 @@ export class LockedMouseEvents implements IMouseEvents { const videoElementParent = this.videoElementProvider.getVideoParentElement(); const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; if ( document.pointerLockElement === videoElementParent || @@ -90,7 +90,7 @@ export class LockedMouseEvents implements IMouseEvents { }); newKeysIterable.forEach((uniqueKeycode) => { - toStreamerHandlers.get('KeyUp')('KeyUp', [uniqueKeycode]); + toStreamerHandlers.get('KeyUp')([uniqueKeycode]); }); // Reset the active keys back to nothing activeKeys = []; @@ -106,7 +106,7 @@ export class LockedMouseEvents implements IMouseEvents { return; } const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; const styleWidth = this.videoElementProvider.getVideoParentElement().clientWidth; const styleHeight = @@ -138,7 +138,7 @@ export class LockedMouseEvents implements IMouseEvents { mouseEvent.movementX, mouseEvent.movementY ); - toStreamerHandlers.get('MouseMove')('MouseMove', [ + toStreamerHandlers.get('MouseMove')([ this.coord.x, this.coord.y, delta.x, @@ -156,11 +156,11 @@ export class LockedMouseEvents implements IMouseEvents { } const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseDown')('MouseDown', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseDown')([ mouseEvent.button, - // We use the store value of this.coord as opposed to the mouseEvent.x/y as the mouseEvent location - // uses the system cursor location which hasn't moved + // We use the store value of this.coord as opposed to the mouseEvent.x/y as the mouseEvent location + // uses the system cursor location which hasn't moved this.coord.x, this.coord.y ]); @@ -175,11 +175,11 @@ export class LockedMouseEvents implements IMouseEvents { return; } const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseUp')('MouseUp', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseUp')([ mouseEvent.button, - // We use the store value of this.coord as opposed to the mouseEvent.x/y as the mouseEvent location - // uses the system cursor location which hasn't moved + // We use the store value of this.coord as opposed to the mouseEvent.x/y as the mouseEvent location + // uses the system cursor location which hasn't moved this.coord.x, this.coord.y ]); @@ -194,11 +194,11 @@ export class LockedMouseEvents implements IMouseEvents { return; } const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseWheel')('MouseWheel', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseWheel')([ wheelEvent.wheelDelta, - // We use the store value of this.coord as opposed to the mouseEvent.x/y as the mouseEvent location - // uses the system cursor location which hasn't moved + // We use the store value of this.coord as opposed to the mouseEvent.x/y as the mouseEvent location + // uses the system cursor location which hasn't moved this.coord.x, this.coord.y ]); @@ -213,11 +213,11 @@ export class LockedMouseEvents implements IMouseEvents { return; } const toStreamerHandlers = - this.mouseController.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseDouble')('MouseDouble', [ + this.mouseController.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseDouble')([ mouseEvent.button, - // We use the store value of this.coord as opposed to the mouseEvent.x/y as the mouseEvent location - // uses the system cursor location which hasn't moved + // We use the store value of this.coord as opposed to the mouseEvent.x/y as the mouseEvent location + // uses the system cursor location which hasn't moved this.coord.x, this.coord.y ]); diff --git a/Frontend/library/src/Inputs/MouseController.ts b/Frontend/library/src/Inputs/MouseController.ts index 2a665618..14f14a2c 100644 --- a/Frontend/library/src/Inputs/MouseController.ts +++ b/Frontend/library/src/Inputs/MouseController.ts @@ -139,8 +139,8 @@ export class MouseController { return; } const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseEnter')('MouseEnter'); + this.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseEnter')(); } /** @@ -151,8 +151,8 @@ export class MouseController { return; } const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseLeave')('MouseLeave'); + this.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseLeave')(); } /** @@ -171,8 +171,8 @@ export class MouseController { 6 ); const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseDown')('MouseDown', [button, X, Y]); + this.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseDown')([button, X, Y]); } /** @@ -195,11 +195,7 @@ export class MouseController { Y ); const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); - toStreamerHandlers.get('MouseUp')('MouseUp', [ - button, - coord.x, - coord.y - ]); + this.toStreamerMessagesProvider.toStreamerHandlers; + toStreamerHandlers.get('MouseUp')([button, coord.x, coord.y]); } } diff --git a/Frontend/library/src/Inputs/TouchController.ts b/Frontend/library/src/Inputs/TouchController.ts index 7e1cea36..ff8209cd 100644 --- a/Frontend/library/src/Inputs/TouchController.ts +++ b/Frontend/library/src/Inputs/TouchController.ts @@ -128,7 +128,7 @@ export class TouchController implements ITouchController { const videoElementParent = this.videoElementProvider.getVideoParentElement(); const toStreamerHandlers = - this.toStreamerMessagesProvider.getToStreamHandlersMap(); + this.toStreamerMessagesProvider.toStreamerHandlers; for (let t = 0; t < touches.length; t++) { const numTouches = 1; // the number of touches to be sent this message @@ -147,7 +147,7 @@ export class TouchController implements ITouchController { ); switch (type) { case 'TouchStart': - toStreamerHandlers.get('TouchStart')('TouchStart', [ + toStreamerHandlers.get('TouchStart')([ numTouches, coord.x, coord.y, @@ -157,7 +157,7 @@ export class TouchController implements ITouchController { ]); break; case 'TouchEnd': - toStreamerHandlers.get('TouchEnd')('TouchEnd', [ + toStreamerHandlers.get('TouchEnd')([ numTouches, coord.x, coord.y, @@ -167,7 +167,7 @@ export class TouchController implements ITouchController { ]); break; case 'TouchMove': - toStreamerHandlers.get('TouchMove')('TouchMove', [ + toStreamerHandlers.get('TouchMove')([ numTouches, coord.x, coord.y, diff --git a/Frontend/library/src/Inputs/XRGamepadController.ts b/Frontend/library/src/Inputs/XRGamepadController.ts new file mode 100644 index 00000000..cc8ed6b6 --- /dev/null +++ b/Frontend/library/src/Inputs/XRGamepadController.ts @@ -0,0 +1,125 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController'; +import { Controller } from './GamepadTypes'; +import { WebXRUtils } from '../Util/WebXRUtils'; + +/** + * The class that handles the functionality of xrgamepads and controllers + */ +export class XRGamepadController { + controllers: Array; + toStreamerMessagesProvider: StreamMessageController; + + /** + * @param toStreamerMessagesProvider - Stream message instance + */ + constructor(toStreamerMessagesProvider: StreamMessageController) { + this.toStreamerMessagesProvider = toStreamerMessagesProvider; + this.controllers = []; + } + + updateStatus( + source: XRInputSource, + frame: XRFrame, + refSpace: XRReferenceSpace + ) { + if (source.gamepad) { + const gamepadPose = frame.getPose(source.gripSpace, refSpace); + if (!gamepadPose) { + return; + } + + let system = 0; + if (source.profiles.includes('htc-vive')) { + system = 1; + } else if (source.profiles.includes('oculus-touch')) { + system = 2; + } + // TODO (william.belcher): Add other profiles (Quest, Microsoft Mixed Reality, etc) + this.toStreamerMessagesProvider.toStreamerHandlers.get('XRSystem')([ + system + ]); + + // Default: AnyHand (2) + let handedness = 2; + switch (source.handedness) { + case 'left': + handedness = 0; + break; + case 'right': + handedness = 1; + break; + } + + // Send controller transform + const matrix = gamepadPose.transform.matrix; + const mat = []; + for (let i = 0; i < 16; i++) { + mat[i] = new Float32Array([matrix[i]])[0]; + } + + // prettier-ignore + this.toStreamerMessagesProvider.toStreamerHandlers.get('XRControllerTransform')([ + mat[0], mat[4], mat[8], mat[12], + mat[1], mat[5], mat[9], mat[13], + mat[2], mat[6], mat[10], mat[14], + mat[3], mat[7], mat[11], mat[15], + handedness + ]); + + // Handle controller buttons and axes + if (this.controllers[handedness] === undefined) { + this.controllers[handedness] = { + prevState: undefined, + currentState: undefined + }; + this.controllers[handedness].prevState = + WebXRUtils.deepCopyGamepad(source.gamepad); + } + + this.controllers[handedness].currentState = + WebXRUtils.deepCopyGamepad(source.gamepad); + + const controller = this.controllers[handedness]; + const currState = controller.currentState; + const prevState = controller.prevState; + // Iterate over buttons + for (let i = 0; i < currState.buttons.length; i++) { + const currButton = currState.buttons[i]; + const prevButton = prevState.buttons[i]; + + if (currButton.pressed) { + // press + this.toStreamerMessagesProvider.toStreamerHandlers.get( + 'XRButtonPressed' + )([handedness, i, prevButton.pressed ? 1 : 0]); + } else if (!currButton.pressed && prevButton.pressed) { + this.toStreamerMessagesProvider.toStreamerHandlers.get( + 'XRButtonReleased' + )([handedness, i, 0]); + } + + if (currButton.touched && !currButton.pressed) { + // press + this.toStreamerMessagesProvider.toStreamerHandlers.get( + 'XRButtonPressed' + )([handedness, 3, prevButton.touched ? 1 : 0]); + } else if (!currButton.touched && prevButton.touched) { + this.toStreamerMessagesProvider.toStreamerHandlers.get( + 'XRButtonReleased' + )([handedness, 3, 0]); + } + } + + // Iterate over gamepad axes + for (let i = 0; i < currState.axes.length; i++) { + this.toStreamerMessagesProvider.toStreamerHandlers.get( + 'XRAnalog' + )([handedness, i, currState.axes[i]]); + } + + this.controllers[handedness].prevState = currState; + } + } +} diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index 7803db9f..fa173d7b 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -23,14 +23,14 @@ export class AggregatedStats { inboundVideoStats: InboundVideoStats; inboundAudioStats: InboundAudioStats; lastVideoStats: InboundVideoStats; - lastAudioStats: InboundAudioStats; + lastAudioStats: InboundAudioStats; candidatePair: CandidatePairStats; DataChannelStats: DataChannelStats; localCandidates: Array; remoteCandidates: Array; outBoundVideoStats: OutBoundVideoStats; streamStats: StreamStats; - codecs: Map; + codecs: Map; constructor() { this.inboundVideoStats = new InboundVideoStats(); @@ -39,7 +39,7 @@ export class AggregatedStats { this.DataChannelStats = new DataChannelStats(); this.outBoundVideoStats = new OutBoundVideoStats(); this.streamStats = new StreamStats(); - this.codecs = new Map; + this.codecs = new Map(); } /** @@ -60,7 +60,7 @@ export class AggregatedStats { case 'certificate': break; case 'codec': - this.handleCodec(stat); + this.handleCodec(stat); break; case 'data-channel': this.handleDataChannel(stat); @@ -182,40 +182,40 @@ export class AggregatedStats { handleInBoundRTP(stat: InboundRTPStats) { switch (stat.kind) { case 'video': - // Need to convert to unknown first to remove an error around - // InboundVideoStats having the bitrate member which isn't found on - // the InboundRTPStats - this.inboundVideoStats = stat as unknown as InboundVideoStats; + // Need to convert to unknown first to remove an error around + // InboundVideoStats having the bitrate member which isn't found on + // the InboundRTPStats + this.inboundVideoStats = stat as unknown as InboundVideoStats; if (this.lastVideoStats != undefined) { this.inboundVideoStats.bitrate = (8 * (this.inboundVideoStats.bytesReceived - this.lastVideoStats.bytesReceived)) / - (this.inboundVideoStats.timestamp - this.lastVideoStats.timestamp); + (this.inboundVideoStats.timestamp - + this.lastVideoStats.timestamp); this.inboundVideoStats.bitrate = Math.floor( this.inboundVideoStats.bitrate ); - } this.lastVideoStats = { ...this.inboundVideoStats }; break; case 'audio': - // Need to convert to unknown first to remove an error around - // InboundAudioStats having the bitrate member which isn't found on - // the InboundRTPStats - this.inboundAudioStats = stat as unknown as InboundAudioStats; + // Need to convert to unknown first to remove an error around + // InboundAudioStats having the bitrate member which isn't found on + // the InboundRTPStats + this.inboundAudioStats = stat as unknown as InboundAudioStats; - if (this.lastAudioStats != undefined) { + if (this.lastAudioStats != undefined) { this.inboundAudioStats.bitrate = (8 * (this.inboundAudioStats.bytesReceived - this.lastAudioStats.bytesReceived)) / - (this.inboundAudioStats.timestamp - this.lastAudioStats.timestamp); + (this.inboundAudioStats.timestamp - + this.lastAudioStats.timestamp); this.inboundAudioStats.bitrate = Math.floor( this.inboundAudioStats.bitrate ); - } this.lastAudioStats = { ...this.inboundAudioStats }; break; @@ -264,11 +264,15 @@ export class AggregatedStats { } } - handleCodec(stat: CodecStats) { - const codecId = stat.id; - const codecType = `${stat.mimeType.replace("video/", "").replace("audio/", "")}${(stat.sdpFmtpLine) ? ` ${stat.sdpFmtpLine}` : ""}`; - this.codecs.set(codecId, codecType); - } + handleCodec(stat: CodecStats) { + const codecId = stat.id; + const codecType = `${stat.mimeType + .replace('video/', '') + .replace('audio/', '')}${ + stat.sdpFmtpLine ? ` ${stat.sdpFmtpLine}` : '' + }`; + this.codecs.set(codecId, codecType); + } /** * Check if a value coming in from our stats is actually a number diff --git a/Frontend/library/src/PeerConnectionController/CodecStats.ts b/Frontend/library/src/PeerConnectionController/CodecStats.ts index 76ccdfbe..75fc9da4 100644 --- a/Frontend/library/src/PeerConnectionController/CodecStats.ts +++ b/Frontend/library/src/PeerConnectionController/CodecStats.ts @@ -4,16 +4,16 @@ * Codec Stats collected from the RTC Stats Report */ export class CodecStats { - /* common stats */ - clockRate: number; - id: string; - mimeType: string; - payloadType: number; - sdpFmtpLine: string; - timestamp: number; - transportId: string; - type: string; + /* common stats */ + clockRate: number; + id: string; + mimeType: string; + payloadType: number; + sdpFmtpLine: string; + timestamp: number; + transportId: string; + type: string; - /* audio specific stats */ - channels: number; -} \ No newline at end of file + /* audio specific stats */ + channels: number; +} diff --git a/Frontend/library/src/PeerConnectionController/InboundRTPStats.ts b/Frontend/library/src/PeerConnectionController/InboundRTPStats.ts index 411a7f54..030b9d43 100644 --- a/Frontend/library/src/PeerConnectionController/InboundRTPStats.ts +++ b/Frontend/library/src/PeerConnectionController/InboundRTPStats.ts @@ -5,40 +5,40 @@ */ export class InboundAudioStats { audioLevel: number; - bytesReceived: number; - codecId: string - concealedSamples: number - concealmentEvents: number; - fecPacketsDiscarded: number; - fecPacketsReceived: number - headerBytesReceived: number; - id: string; - insertedSamplesForDeceleration: number; - jitter: number; - jitterBufferDelay: number; - jitterBufferEmittedCount: number - jitterBufferMinimumDelay: number; - jitterBufferTargetDelay: number; - kind: string - lastPacketReceivedTimestamp: number; - mediaType: string; - mid: string; - packetsDiscarded: number; - packetsLost: number; - packetsReceived: number; - removedSamplesForAcceleration: number; - silentConcealedSamples: number; - ssrc: number; - timestamp: number; - totalAudioEnergy: number; - totalSamplesDuration: number; - totalSamplesReceived: number; - trackIdentifier: string; - transportId: string; - type: string; + bytesReceived: number; + codecId: string; + concealedSamples: number; + concealmentEvents: number; + fecPacketsDiscarded: number; + fecPacketsReceived: number; + headerBytesReceived: number; + id: string; + insertedSamplesForDeceleration: number; + jitter: number; + jitterBufferDelay: number; + jitterBufferEmittedCount: number; + jitterBufferMinimumDelay: number; + jitterBufferTargetDelay: number; + kind: string; + lastPacketReceivedTimestamp: number; + mediaType: string; + mid: string; + packetsDiscarded: number; + packetsLost: number; + packetsReceived: number; + removedSamplesForAcceleration: number; + silentConcealedSamples: number; + ssrc: number; + timestamp: number; + totalAudioEnergy: number; + totalSamplesDuration: number; + totalSamplesReceived: number; + trackIdentifier: string; + transportId: string; + type: string; - /* additional, custom stats */ - bitrate: number; + /* additional, custom stats */ + bitrate: number; } /** @@ -46,109 +46,109 @@ export class InboundAudioStats { */ export class InboundVideoStats { bytesReceived: number; - codecId: string; - firCount: number; - frameHeight: number; - frameWidth: number; - framesAssembledFromMultiplePackets: number; - framesDecoded: number; - framesDropped: number; - framesPerSecond: number; - framesReceived: number; - freezeCount: number; - googTimingFrameInfo: string; - headerBytesReceived: number; - id: string; - jitter: number; - jitterBufferDelay: number; - jitterBufferEmittedCount: number; - keyFramesDecoded: number; - kind: string; - lastPacketReceivedTimestamp: number; - mediaType: string; - mid: string; - nackCount: number; - packetsLost: number; - packetsReceived: number; - pauseCount: number; - pliCount: number; - ssrc: number; - timestamp: number; - totalAssemblyTime: number; - totalDecodeTime: number; - totalFreezesDuration: number; - totalInterFrameDelay: number; - totalPausesDuration: number; - totalProcessingDelay: number; - totalSquaredInterFrameDelay: number; - trackIdentifier: string; - transportId: string; - type: string; + codecId: string; + firCount: number; + frameHeight: number; + frameWidth: number; + framesAssembledFromMultiplePackets: number; + framesDecoded: number; + framesDropped: number; + framesPerSecond: number; + framesReceived: number; + freezeCount: number; + googTimingFrameInfo: string; + headerBytesReceived: number; + id: string; + jitter: number; + jitterBufferDelay: number; + jitterBufferEmittedCount: number; + keyFramesDecoded: number; + kind: string; + lastPacketReceivedTimestamp: number; + mediaType: string; + mid: string; + nackCount: number; + packetsLost: number; + packetsReceived: number; + pauseCount: number; + pliCount: number; + ssrc: number; + timestamp: number; + totalAssemblyTime: number; + totalDecodeTime: number; + totalFreezesDuration: number; + totalInterFrameDelay: number; + totalPausesDuration: number; + totalProcessingDelay: number; + totalSquaredInterFrameDelay: number; + trackIdentifier: string; + transportId: string; + type: string; - /* additional, custom stats */ - bitrate: number; + /* additional, custom stats */ + bitrate: number; } /** * Inbound Stats collected from the RTC Stats Report */ export class InboundRTPStats { - /* common stats */ - bytesReceived: number; - codecId: string; - headerBytesReceived: number; - id: string; - jitter: number; - jitterBufferDelay: number; - jitterBufferEmittedCount: number; - kind: string; - lastPacketReceivedTimestamp: number; - mediaType: string; - mid: string; - packetsLost: number; - packetsReceived: number; - ssrc: number; - timestamp: number; - trackIdentifier: string; - transportId: string; - type: string; + /* common stats */ + bytesReceived: number; + codecId: string; + headerBytesReceived: number; + id: string; + jitter: number; + jitterBufferDelay: number; + jitterBufferEmittedCount: number; + kind: string; + lastPacketReceivedTimestamp: number; + mediaType: string; + mid: string; + packetsLost: number; + packetsReceived: number; + ssrc: number; + timestamp: number; + trackIdentifier: string; + transportId: string; + type: string; - /* audio specific stats */ - audioLevel: number; - concealedSamples: number - concealmentEvents: number; - fecPacketsDiscarded: number; - fecPacketsReceived: number - insertedSamplesForDeceleration: number; - jitterBufferMinimumDelay: number; - jitterBufferTargetDelay: number; - packetsDiscarded: number; - removedSamplesForAcceleration: number; - silentConcealedSamples: number; - totalAudioEnergy: number; - totalSamplesDuration: number; - totalSamplesReceived: number; + /* audio specific stats */ + audioLevel: number; + concealedSamples: number; + concealmentEvents: number; + fecPacketsDiscarded: number; + fecPacketsReceived: number; + insertedSamplesForDeceleration: number; + jitterBufferMinimumDelay: number; + jitterBufferTargetDelay: number; + packetsDiscarded: number; + removedSamplesForAcceleration: number; + silentConcealedSamples: number; + totalAudioEnergy: number; + totalSamplesDuration: number; + totalSamplesReceived: number; - /* video specific stats */ - firCount: number; - frameHeight: number; - frameWidth: number; - framesAssembledFromMultiplePackets: number; - framesDecoded: number; - framesDropped: number; - framesPerSecond: number; - framesReceived: number; - freezeCount: number; - googTimingFrameInfo: string; - keyFramesDecoded: number; - nackCount: number; - pauseCount: number; - pliCount: number; - totalAssemblyTime: number; - totalDecodeTime: number; - totalFreezesDuration: number; - totalInterFrameDelay: number; - totalPausesDuration: number; - totalProcessingDelay: number; - totalSquaredInterFrameDelay: number; + /* video specific stats */ + firCount: number; + frameHeight: number; + frameWidth: number; + framesAssembledFromMultiplePackets: number; + framesDecoded: number; + framesDropped: number; + framesPerSecond: number; + framesReceived: number; + freezeCount: number; + googTimingFrameInfo: string; + keyFramesDecoded: number; + nackCount: number; + pauseCount: number; + pliCount: number; + totalAssemblyTime: number; + totalDecodeTime: number; + totalFreezesDuration: number; + totalInterFrameDelay: number; + totalPausesDuration: number; + totalProcessingDelay: number; + totalSquaredInterFrameDelay: number; } diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index 5cc5bbc6..0472b8fd 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -12,15 +12,19 @@ export class PeerConnectionController { peerConnection: RTCPeerConnection; aggregatedStats: AggregatedStats; config: Config; - preferredCodec: string; - updateCodecSelection: boolean; + preferredCodec: string; + updateCodecSelection: boolean; /** * Create a new RTC Peer Connection client * @param options - Peer connection Options * @param config - The config for our PS experience. */ - constructor(options: RTCConfiguration, config: Config, preferredCodec: string) { + constructor( + options: RTCConfiguration, + config: Config, + preferredCodec: string + ) { this.config = config; // Set the ICE transport to relay if TURN enabled @@ -47,8 +51,8 @@ export class PeerConnectionController { this.peerConnection.ondatachannel = (ev: RTCDataChannelEvent) => this.handleDataChannel(ev); this.aggregatedStats = new AggregatedStats(); - this.preferredCodec = preferredCodec; - this.updateCodecSelection = true; + this.preferredCodec = preferredCodec; + this.updateCodecSelection = true; } /** @@ -135,8 +139,15 @@ export class PeerConnectionController { }); }); - // Ugly syntax, but this achieves the intersection of the browser supported list and the UE supported list - this.config.setOptionSettingOptions(OptionParameters.PreferredCodec, this.parseAvailableCodecs(offer).filter(value => this.config.getSettingOption(OptionParameters.PreferredCodec).options.includes(value))); + // Ugly syntax, but this achieves the intersection of the browser supported list and the UE supported list + this.config.setOptionSettingOptions( + OptionParameters.PreferredCodec, + this.parseAvailableCodecs(offer).filter((value) => + this.config + .getSettingOption(OptionParameters.PreferredCodec) + .options.includes(value) + ) + ); } /** @@ -145,8 +156,15 @@ export class PeerConnectionController { */ receiveAnswer(answer: RTCSessionDescriptionInit) { this.peerConnection.setRemoteDescription(answer); - // Ugly syntax, but this achieves the intersection of the browser supported list and the UE supported list - this.config.setOptionSettingOptions(OptionParameters.PreferredCodec, this.parseAvailableCodecs(answer).filter(value => this.config.getSettingOption(OptionParameters.PreferredCodec).options.includes(value))); + // Ugly syntax, but this achieves the intersection of the browser supported list and the UE supported list + this.config.setOptionSettingOptions( + OptionParameters.PreferredCodec, + this.parseAvailableCodecs(answer).filter((value) => + this.config + .getSettingOption(OptionParameters.PreferredCodec) + .options.includes(value) + ) + ); } /** @@ -157,10 +175,14 @@ export class PeerConnectionController { this.aggregatedStats.processStats(StatsData); this.onVideoStats(this.aggregatedStats); - // Update the preferred codec selection based on what was actually negotiated - if(this.updateCodecSelection) { - this.config.getSettingOption(OptionParameters.PreferredCodec).selected = this.aggregatedStats.codecs.get(this.aggregatedStats.inboundVideoStats.codecId) - } + // Update the preferred codec selection based on what was actually negotiated + if (this.updateCodecSelection) { + this.config.getSettingOption( + OptionParameters.PreferredCodec + ).selected = this.aggregatedStats.codecs.get( + this.aggregatedStats.inboundVideoStats.codecId + ); + } }); } @@ -198,7 +220,7 @@ export class PeerConnectionController { } // Force mono or stereo based on whether ?forceMono was passed or not - audioSDP += this.config.isFlagEnabled(Flags.ForceMonoAudio) + audioSDP += this.config.isFlagEnabled(Flags.ForceMonoAudio) ? 'stereo=0;' : 'stereo=1;'; @@ -331,43 +353,56 @@ export class PeerConnectionController { // Setup a transceiver for getting UE video this.peerConnection.addTransceiver('video', { direction: 'recvonly' }); - // We can only set preferrec codec on Chrome - if(RTCRtpReceiver.getCapabilities && this.preferredCodec != "") { - for(const transceiver of this.peerConnection.getTransceivers()) { - if(transceiver && transceiver.receiver && transceiver.receiver.track && transceiver.receiver.track.kind === "video") - { - const preferredRTPCodec = this.preferredCodec.split(" "); - const codecs = [{ - mimeType: 'video/' + preferredRTPCodec[0] /* Name */, - clockRate: 90000, - sdpFmtpLine: preferredRTPCodec[1] /* sdpFmtpLine */ ? preferredRTPCodec[1] : '' - }]; - - this.config.getSettingOption(OptionParameters.PreferredCodec).options - .filter((option) => { - // Remove the preferred codec from the list of possible codecs as we've set it already - return option != this.preferredCodec; - }).forEach((option) => { - // Ammend the rest of the browsers supported codecs - const altCodec = option.split(" "); - codecs.push({ - mimeType: 'video/' + altCodec[0] /* Name */, - clockRate: 90000, - sdpFmtpLine: altCodec[1] /* sdpFmtpLine */ ? altCodec[1] : '' - }) - }) - - for(const codec of codecs) { - if(codec.sdpFmtpLine === '') { - // We can't dynamically add members to the codec, so instead remove the field if it's empty - delete codec.sdpFmtpLine; - } - } - - transceiver.setCodecPreferences(codecs); - } - } - } + // We can only set preferrec codec on Chrome + if (RTCRtpReceiver.getCapabilities && this.preferredCodec != '') { + for (const transceiver of this.peerConnection.getTransceivers()) { + if ( + transceiver && + transceiver.receiver && + transceiver.receiver.track && + transceiver.receiver.track.kind === 'video' + ) { + const preferredRTPCodec = this.preferredCodec.split(' '); + const codecs = [ + { + mimeType: + 'video/' + preferredRTPCodec[0] /* Name */, + clockRate: 90000, + sdpFmtpLine: preferredRTPCodec[1] /* sdpFmtpLine */ + ? preferredRTPCodec[1] + : '' + } + ]; + + this.config + .getSettingOption(OptionParameters.PreferredCodec) + .options.filter((option) => { + // Remove the preferred codec from the list of possible codecs as we've set it already + return option != this.preferredCodec; + }) + .forEach((option) => { + // Ammend the rest of the browsers supported codecs + const altCodec = option.split(' '); + codecs.push({ + mimeType: 'video/' + altCodec[0] /* Name */, + clockRate: 90000, + sdpFmtpLine: altCodec[1] /* sdpFmtpLine */ + ? altCodec[1] + : '' + }); + }); + + for (const codec of codecs) { + if (codec.sdpFmtpLine === '') { + // We can't dynamically add members to the codec, so instead remove the field if it's empty + delete codec.sdpFmtpLine; + } + } + + transceiver.setCodecPreferences(codecs); + } + } + } // Setup a transceiver for sending mic audio to UE and receiving audio from UE if (!useMic) { @@ -474,34 +509,47 @@ export class PeerConnectionController { // Default Functionality: Do Nothing } - parseAvailableCodecs(rtcSessionDescription: RTCSessionDescriptionInit): Array { - // No point in updating the available codecs if on FF - if(!RTCRtpReceiver.getCapabilities) return ["Only available on Chrome"]; - - const ueSupportedCodecs: Array = []; - const sections = splitSections(rtcSessionDescription.sdp); - // discard the session information as we only want media related info - sections.shift(); - sections.forEach(mediaSection => { - const {codecs} = parseRtpParameters(mediaSection); - // Filter only for VPX / H26X / AV1 - const matcher = /(VP\d|H26\d|AV1).*/ - codecs.forEach(c => { - const str = c.name + ' ' + Object.keys(c.parameters || {}).map(p => p + '=' + c.parameters[p]).join(';'); - const match = matcher.exec(str); - if(match !== null) { - if(c.name == "VP9") { - // UE answers don't specify profile but we know we want profile 0 - c.parameters = { - "profile-id": "0" - } - } - const codecStr: string = c.name + " " + Object.keys(c.parameters || {}).map(p => p + '=' + c.parameters[p]).join(';'); - ueSupportedCodecs.push(codecStr); - } - }) - }); - - return ueSupportedCodecs; - } + parseAvailableCodecs( + rtcSessionDescription: RTCSessionDescriptionInit + ): Array { + // No point in updating the available codecs if on FF + if (!RTCRtpReceiver.getCapabilities) + return ['Only available on Chrome']; + + const ueSupportedCodecs: Array = []; + const sections = splitSections(rtcSessionDescription.sdp); + // discard the session information as we only want media related info + sections.shift(); + sections.forEach((mediaSection) => { + const { codecs } = parseRtpParameters(mediaSection); + // Filter only for VPX / H26X / AV1 + const matcher = /(VP\d|H26\d|AV1).*/; + codecs.forEach((c) => { + const str = + c.name + + ' ' + + Object.keys(c.parameters || {}) + .map((p) => p + '=' + c.parameters[p]) + .join(';'); + const match = matcher.exec(str); + if (match !== null) { + if (c.name == 'VP9') { + // UE answers don't specify profile but we know we want profile 0 + c.parameters = { + 'profile-id': '0' + }; + } + const codecStr: string = + c.name + + ' ' + + Object.keys(c.parameters || {}) + .map((p) => p + '=' + c.parameters[p]) + .join(';'); + ueSupportedCodecs.push(codecStr); + } + }); + }); + + return ueSupportedCodecs; + } } diff --git a/Frontend/library/src/UI/Controls.ts b/Frontend/library/src/UI/Controls.ts index 8a8fd00b..295d3320 100644 --- a/Frontend/library/src/UI/Controls.ts +++ b/Frontend/library/src/UI/Controls.ts @@ -3,6 +3,8 @@ import { FullScreenIcon } from './FullscreenIcon'; import { SettingsIcon } from './SettingsIcon'; import { StatsIcon } from './StatsIcon'; +import { XRIcon } from './XRIcon'; +import { WebXRController } from '../WebXR/WebXRController'; /** * Element containing various controls like stats, settings, fullscreen. @@ -11,6 +13,7 @@ export class Controls { statsIcon: StatsIcon; fullscreenIcon: FullScreenIcon; settingsIcon: SettingsIcon; + xrIcon: XRIcon; _rootElement: HTMLElement; @@ -21,6 +24,7 @@ export class Controls { this.statsIcon = new StatsIcon(); this.settingsIcon = new SettingsIcon(); this.fullscreenIcon = new FullScreenIcon(); + this.xrIcon = new XRIcon(); } /** @@ -33,6 +37,13 @@ export class Controls { this._rootElement.appendChild(this.fullscreenIcon.rootElement); this._rootElement.appendChild(this.settingsIcon.rootElement); this._rootElement.appendChild(this.statsIcon.rootElement); + WebXRController.isSessionSupported('immersive-vr').then( + (supported: boolean) => { + if (supported) { + this._rootElement.appendChild(this.xrIcon.rootElement); + } + } + ); } return this._rootElement; } diff --git a/Frontend/library/src/UI/StatsPanel.ts b/Frontend/library/src/UI/StatsPanel.ts index 5914e086..5b3a26e6 100644 --- a/Frontend/library/src/UI/StatsPanel.ts +++ b/Frontend/library/src/UI/StatsPanel.ts @@ -179,22 +179,22 @@ export class StatsPanel { ); // Bitrate - if(stats.inboundVideoStats.bitrate) { - this.addOrUpdateStat( - 'VideoBitrateStat', - 'Video Bitrate (kbps)', - stats.inboundVideoStats.bitrate.toString() - ); - } - - if(stats.inboundAudioStats.bitrate) { - this.addOrUpdateStat( - 'AudioBitrateStat', - 'Audio Bitrate (kbps)', - stats.inboundAudioStats.bitrate.toString() - ); - } - + if (stats.inboundVideoStats.bitrate) { + this.addOrUpdateStat( + 'VideoBitrateStat', + 'Video Bitrate (kbps)', + stats.inboundVideoStats.bitrate.toString() + ); + } + + if (stats.inboundAudioStats.bitrate) { + this.addOrUpdateStat( + 'AudioBitrateStat', + 'Audio Bitrate (kbps)', + stats.inboundAudioStats.bitrate.toString() + ); + } + // Video resolution const resStat = Object.prototype.hasOwnProperty.call( @@ -227,13 +227,13 @@ export class StatsPanel { ); // Framerate - if(stats.inboundVideoStats.framesPerSecond) { - this.addOrUpdateStat( - 'FramerateStat', - 'Framerate', - stats.inboundVideoStats.framesPerSecond.toString() - ); - } + if (stats.inboundVideoStats.framesPerSecond) { + this.addOrUpdateStat( + 'FramerateStat', + 'Framerate', + stats.inboundVideoStats.framesPerSecond.toString() + ); + } // Frames dropped this.addOrUpdateStat( @@ -242,24 +242,23 @@ export class StatsPanel { stats.inboundVideoStats.framesDropped.toString() ); - if(stats.inboundVideoStats.codecId) { - this.addOrUpdateStat( - 'VideoCodecStat', - 'Video codec', - // Split the codec to remove the Fmtp line - stats.codecs.get(stats.inboundVideoStats.codecId).split(" ")[0] - ); - } - - if(stats.inboundAudioStats.codecId) { - this.addOrUpdateStat( - 'AudioCodecStat', - 'Audio codec', - // Split the codec to remove the Fmtp line - stats.codecs.get(stats.inboundAudioStats.codecId).split(" ")[0] - ); - } - + if (stats.inboundVideoStats.codecId) { + this.addOrUpdateStat( + 'VideoCodecStat', + 'Video codec', + // Split the codec to remove the Fmtp line + stats.codecs.get(stats.inboundVideoStats.codecId).split(' ')[0] + ); + } + + if (stats.inboundAudioStats.codecId) { + this.addOrUpdateStat( + 'AudioCodecStat', + 'Audio codec', + // Split the codec to remove the Fmtp line + stats.codecs.get(stats.inboundAudioStats.codecId).split(' ')[0] + ); + } // RTT const netRTT = diff --git a/Frontend/library/src/UI/XRIcon.ts b/Frontend/library/src/UI/XRIcon.ts new file mode 100644 index 00000000..4f469fc9 --- /dev/null +++ b/Frontend/library/src/UI/XRIcon.ts @@ -0,0 +1,70 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +/** + * XR icon that can be clicked. + */ +export class XRIcon { + _rootElement: HTMLButtonElement; + _xrIcon: SVGElement; + _tooltipText: HTMLElement; + + /** + * Get the the button containing the XR icon. + */ + public get rootElement(): HTMLButtonElement { + if (!this._rootElement) { + this._rootElement = document.createElement('button'); + this._rootElement.type = 'button'; + this._rootElement.classList.add('UiTool'); + this._rootElement.id = 'xrBtn'; + this._rootElement.appendChild(this.xrIcon); + this._rootElement.appendChild(this.tooltipText); + } + return this._rootElement; + } + + public get tooltipText(): HTMLElement { + if (!this._tooltipText) { + this._tooltipText = document.createElement('span'); + this._tooltipText.classList.add('tooltiptext'); + this._tooltipText.innerHTML = 'XR'; + } + return this._tooltipText; + } + + public get xrIcon(): SVGElement { + if (!this._xrIcon) { + this._xrIcon = document.createElementNS( + 'http://www.w3.org/2000/svg', + 'svg' + ); + this._xrIcon.setAttributeNS(null, 'id', 'xrIcon'); + this._xrIcon.setAttributeNS(null, 'x', '0px'); + this._xrIcon.setAttributeNS(null, 'y', '0px'); + this._xrIcon.setAttributeNS(null, 'viewBox', '0 0 100 100'); + + // create svg group for the paths + const svgGroup = document.createElementNS( + 'http://www.w3.org/2000/svg', + 'g' + ); + svgGroup.classList.add('svgIcon'); + this._xrIcon.appendChild(svgGroup); + + // create paths for the icon itself, the path of the xr headset + const path = document.createElementNS( + 'http://www.w3.org/2000/svg', + 'path' + ); + + path.setAttributeNS( + null, + 'd', + 'M29 41c-5 0-9 4-9 9s4 9 9 9 9-4 9-9-4-9-9-9zm0 14c-2.8 0-5-2.2-5-5s2.2-5 5-5 5 2.2 5 5-2.2 5-5 5zm42-14c-5 0-9 4-9 9s4 9 9 9 9-4 9-9-4-9-9-9zm0 14c-2.8 0-5-2.2-5-5s2.2-5 5-5 5 2.2 5 5-2.2 5-5 5zm12-31H17c-6.6 0-12 5.4-12 12v28c0 6.6 5.4 12 12 12h14.5c3.5 0 6.8-1.5 9-4.1l3.5-4c1.5-1.7 3.7-2.7 6-2.7s4.5 1 6 2.7l3.5 4c2.3 2.6 5.6 4.1 9 4.1H83c6.6 0 12-5.4 12-12V36c0-6.6-5.4-12-12-12zm8 40c0 4.4-3.6 8-8 8H68.5c-2.3 0-4.5-1-6-2.7l-3.5-4c-2.3-2.6-5.6-4.1-9-4.1-3.5 0-6.8 1.5-9 4.1l-3.5 4C36 71 33.8 72 31.5 72H17c-4.4 0-8-3.6-8-8V36c0-4.4 3.6-8 8-8h66c4.4 0 8 3.6 8 8v28z' + ); + + svgGroup.appendChild(path); + } + return this._xrIcon; + } +} diff --git a/Frontend/library/src/UeInstanceMessage/SendDescriptorController.ts b/Frontend/library/src/UeInstanceMessage/SendDescriptorController.ts index 425504e0..8df32b19 100644 --- a/Frontend/library/src/UeInstanceMessage/SendDescriptorController.ts +++ b/Frontend/library/src/UeInstanceMessage/SendDescriptorController.ts @@ -49,7 +49,7 @@ export class SendDescriptorController { // Convert the descriptor object into a JSON string. const descriptorAsString = JSON.stringify(descriptor); const toStreamerMessages = - this.toStreamerMessagesMapProvider.getToStreamerMessageMap(); + this.toStreamerMessagesMapProvider.toStreamerMessages; const messageFormat = toStreamerMessages.getFromKey(messageType); if (messageFormat === undefined) { Logger.Error( diff --git a/Frontend/library/src/UeInstanceMessage/SendMessageController.ts b/Frontend/library/src/UeInstanceMessage/SendMessageController.ts index 53497814..a6e6dd36 100644 --- a/Frontend/library/src/UeInstanceMessage/SendMessageController.ts +++ b/Frontend/library/src/UeInstanceMessage/SendMessageController.ts @@ -32,7 +32,7 @@ export class SendMessageController { } const toStreamerMessages = - this.toStreamerMessagesMapProvider.getToStreamerMessageMap(); + this.toStreamerMessagesMapProvider.toStreamerMessages; const messageFormat = toStreamerMessages.getFromKey(messageType); if (messageFormat === undefined) { Logger.Error( @@ -66,6 +66,11 @@ export class SendMessageController { byteOffset += 2; break; + case 'float': + data.setFloat32(byteOffset, element, true); + byteOffset += 4; + break; + case 'double': data.setFloat64(byteOffset, element, true); byteOffset += 8; diff --git a/Frontend/library/src/UeInstanceMessage/StreamMessageController.ts b/Frontend/library/src/UeInstanceMessage/StreamMessageController.ts index ef1a9f5b..a822cf5a 100644 --- a/Frontend/library/src/UeInstanceMessage/StreamMessageController.ts +++ b/Frontend/library/src/UeInstanceMessage/StreamMessageController.ts @@ -12,7 +12,7 @@ export class ToStreamerMessage { export class StreamMessageController { toStreamerHandlers: Map< string, - (messageType: string, messageData?: Array | undefined) => void + (messageData?: Array | undefined) => void >; fromStreamerHandlers: Map< string, @@ -30,23 +30,6 @@ export class StreamMessageController { this.fromStreamerMessages = new TwoWayMap(); } - /** - * Get the current map for to streamer handlers - */ - getToStreamHandlersMap(): Map< - string, - (messageType: string, messageData?: Array | undefined) => void - > { - return this.toStreamerHandlers; - } - - /** - * Get the current twoWayMap for to streamer messages - */ - getToStreamerMessageMap(): TwoWayMap { - return this.toStreamerMessages; - } - /** * Populate the Default message protocol */ @@ -237,10 +220,7 @@ export class StreamMessageController { registerMessageHandler( messageDirection: MessageDirection, messageType: string, - messageHandler: ( - messageType: string, - messageData?: unknown | undefined - ) => void + messageHandler: (messageData?: unknown | undefined) => void ) { switch (messageDirection) { case MessageDirection.ToStreamer: diff --git a/Frontend/library/src/Util/WebGLUtils.ts b/Frontend/library/src/Util/WebGLUtils.ts new file mode 100644 index 00000000..d8cbaa2f --- /dev/null +++ b/Frontend/library/src/Util/WebGLUtils.ts @@ -0,0 +1,49 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +export class WebGLUtils { + static vertexShader(): string { + return ` + attribute vec2 a_position; + attribute vec2 a_texCoord; + + // input + uniform vec2 u_resolution; + uniform vec4 u_offset; + + // + varying vec2 v_texCoord; + + void main() { + // convert the rectangle from pixels to 0.0 to 1.0 + vec2 zeroToOne = a_position / u_resolution; + + // convert from 0->1 to 0->2 + vec2 zeroToTwo = zeroToOne * 2.0; + + // convert from 0->2 to -1->+1 (clipspace) + vec2 clipSpace = zeroToTwo - 1.0; + + gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1); + // pass the texCoord to the fragment shader + // The GPU will interpolate this value between points. + v_texCoord = (a_texCoord * u_offset.xy) + u_offset.zw; + } + `; + } + + static fragmentShader(): string { + return ` + precision mediump float; + + // our texture + uniform sampler2D u_image; + + // the texCoords passed in from the vertex shader. + varying vec2 v_texCoord; + + void main() { + gl_FragColor = texture2D(u_image, v_texCoord); + } + `; + } +} diff --git a/Frontend/library/src/Util/WebXRUtils.ts b/Frontend/library/src/Util/WebXRUtils.ts new file mode 100644 index 00000000..6252af2d --- /dev/null +++ b/Frontend/library/src/Util/WebXRUtils.ts @@ -0,0 +1,25 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +export class WebXRUtils { + /** + * Deep copies a gamepad's values by first converting it to a JSON object and then back to a gamepad + * + * @param gamepad the original gamepad + * @returns a new gamepad object, populated with the original gamepads values + */ + static deepCopyGamepad(gamepad: Gamepad): Gamepad { + return JSON.parse( + JSON.stringify({ + buttons: gamepad.buttons.map((b) => + JSON.parse( + JSON.stringify({ + pressed: b.pressed, + touched: b.touched + }) + ) + ), + axes: gamepad.axes + }) + ); + } +} diff --git a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts index b1eb2476..f6b0fb84 100644 --- a/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts +++ b/Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts @@ -88,7 +88,7 @@ export class WebRtcPlayerController { isQualityController: boolean; statsTimerHandle: number; file: FileTemplate; - preferredCodec: string; + preferredCodec: string; // if you override the disconnection message by calling the interface method setDisconnectMessageOverride // it will use this property to store the override message string @@ -181,7 +181,9 @@ export class WebRtcPlayerController { this.webSocketController.onOpen.addEventListener('open', () => { const urlParams = new URLSearchParams(window.location.search); if (urlParams.has(OptionParameters.StreamerId)) { - this.webSocketController.sendSubscribe(urlParams.get(OptionParameters.StreamerId)); + this.webSocketController.sendSubscribe( + urlParams.get(OptionParameters.StreamerId) + ); } else { this.webSocketController.requestStreamerList(); } @@ -224,9 +226,11 @@ export class WebRtcPlayerController { this.isUsingSFU = false; this.isQualityController = false; - this.preferredCodec = ''; + this.preferredCodec = ''; - this.config.addOnOptionSettingChangedListener(OptionParameters.StreamerId, (streamerid) => { + this.config.addOnOptionSettingChangedListener( + OptionParameters.StreamerId, + (streamerid) => { this.webSocketController.sendSubscribe(streamerid); } ); @@ -258,7 +262,6 @@ export class WebRtcPlayerController { message[0] ); this.streamMessageController.fromStreamerHandlers.get(messageType)( - messageType, event.data ); //} catch (e) { @@ -275,27 +278,24 @@ export class WebRtcPlayerController { this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'QualityControlOwnership', - (messageType: string, data: ArrayBuffer) => - this.onQualityControlOwnership(data) + (data: ArrayBuffer) => this.onQualityControlOwnership(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'Response', - (messageType: string, data: ArrayBuffer) => - this.responseController.onResponse(data) + (data: ArrayBuffer) => this.responseController.onResponse(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'Command', - (messageType: string, data: ArrayBuffer) => { + (data: ArrayBuffer) => { this.onCommand(data); } ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'FreezeFrame', - (messageType: string, data: ArrayBuffer) => - this.onFreezeFrameMessage(data) + (data: ArrayBuffer) => this.onFreezeFrameMessage(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, @@ -305,38 +305,32 @@ export class WebRtcPlayerController { this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'VideoEncoderAvgQP', - (messageType: string, data: ArrayBuffer) => - this.handleVideoEncoderAvgQP(data) + (data: ArrayBuffer) => this.handleVideoEncoderAvgQP(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'LatencyTest', - (messageType: string, data: ArrayBuffer) => - this.handleLatencyTestResult(data) + (data: ArrayBuffer) => this.handleLatencyTestResult(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'InitialSettings', - (messageType: string, data: ArrayBuffer) => - this.handleInitialSettings(data) + (data: ArrayBuffer) => this.handleInitialSettings(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'FileExtension', - (messageType: string, data: ArrayBuffer) => - this.onFileExtension(data) + (data: ArrayBuffer) => this.onFileExtension(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'FileMimeType', - (messageType: string, data: ArrayBuffer) => - this.onFileMimeType(data) + (data: ArrayBuffer) => this.onFileMimeType(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'FileContents', - (messageType: string, data: ArrayBuffer) => - this.onFileContents(data) + (data: ArrayBuffer) => this.onFileContents(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, @@ -348,64 +342,73 @@ export class WebRtcPlayerController { this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'InputControlOwnership', - (messageType: string, data: ArrayBuffer) => - this.onInputControlOwnership(data) + (data: ArrayBuffer) => this.onInputControlOwnership(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.FromStreamer, 'Protocol', - (messageType: string, data: ArrayBuffer) => - this.onProtocolMessage(data) + (data: ArrayBuffer) => this.onProtocolMessage(data) ); // To Streamer this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'IFrameRequest', - (messageType: string) => - this.sendMessageController.sendMessageToStreamer(messageType) + () => + this.sendMessageController.sendMessageToStreamer( + 'IFrameRequest' + ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'RequestQualityControl', - (messageType: string) => - this.sendMessageController.sendMessageToStreamer(messageType) + () => + this.sendMessageController.sendMessageToStreamer( + 'RequestQualityControl' + ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'FpsRequest', - (messageType: string) => - this.sendMessageController.sendMessageToStreamer(messageType) + () => this.sendMessageController.sendMessageToStreamer('FpsRequest') ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'AverageBitrateRequest', - (messageType: string) => - this.sendMessageController.sendMessageToStreamer(messageType) + () => + this.sendMessageController.sendMessageToStreamer( + 'AverageBitrateRequest' + ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'StartStreaming', - (messageType: string) => - this.sendMessageController.sendMessageToStreamer(messageType) + () => + this.sendMessageController.sendMessageToStreamer( + 'StartStreaming' + ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'StopStreaming', - (messageType: string) => - this.sendMessageController.sendMessageToStreamer(messageType) + () => + this.sendMessageController.sendMessageToStreamer( + 'StopStreaming' + ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'LatencyTest', - (messageType: string) => - this.sendMessageController.sendMessageToStreamer(messageType) + () => + this.sendMessageController.sendMessageToStreamer('LatencyTest') ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'RequestInitialSettings', - (messageType: string) => - this.sendMessageController.sendMessageToStreamer(messageType) + () => + this.sendMessageController.sendMessageToStreamer( + 'RequestInitialSettings' + ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, @@ -417,156 +420,215 @@ export class WebRtcPlayerController { this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'UIInteraction', - (messageType: string, data: object) => + (data: object) => this.sendDescriptorController.emitUIInteraction(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'Command', - (messageType: string, data: object) => - this.sendDescriptorController.emitCommand(data) + (data: object) => this.sendDescriptorController.emitCommand(data) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'KeyDown', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'KeyDown', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'KeyUp', - (messageType: string, data: Array) => - this.sendMessageController.sendMessageToStreamer( - messageType, - data - ) + (data: Array) => + this.sendMessageController.sendMessageToStreamer('KeyUp', data) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'KeyPress', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'KeyPress', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'MouseEnter', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'MouseEnter', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'MouseLeave', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'MouseLeave', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'MouseDown', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'MouseDown', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'MouseUp', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'MouseUp', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'MouseMove', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'MouseMove', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'MouseWheel', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'MouseWheel', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'MouseDouble', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'MouseDouble', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'TouchStart', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'TouchStart', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'TouchEnd', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'TouchEnd', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'TouchMove', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'TouchMove', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'GamepadButtonPressed', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'GamepadButtonPressed', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'GamepadButtonReleased', - (messageType: string, data: Array) => + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'GamepadButtonReleased', data ) ); this.streamMessageController.registerMessageHandler( MessageDirection.ToStreamer, 'GamepadAnalog', - (messageType: string, data: Array) => + (data: Array) => + this.sendMessageController.sendMessageToStreamer( + 'GamepadAnalog', + data + ) + ); + this.streamMessageController.registerMessageHandler( + MessageDirection.ToStreamer, + 'XRHMDTransform', + (data: Array) => + this.sendMessageController.sendMessageToStreamer( + 'XRHMDTransform', + data + ) + ); + this.streamMessageController.registerMessageHandler( + MessageDirection.ToStreamer, + 'XRControllerTransform', + (data: Array) => + this.sendMessageController.sendMessageToStreamer( + 'XRControllerTransform', + data + ) + ); + this.streamMessageController.registerMessageHandler( + MessageDirection.ToStreamer, + 'XRSystem', + (data: Array) => + this.sendMessageController.sendMessageToStreamer( + 'XRSystem', + data + ) + ); + this.streamMessageController.registerMessageHandler( + MessageDirection.ToStreamer, + 'XRButtonTouched', + (data: Array) => + this.sendMessageController.sendMessageToStreamer( + 'XRButtonTouched', + data + ) + ); + this.streamMessageController.registerMessageHandler( + MessageDirection.ToStreamer, + 'XRButtonPressed', + (data: Array) => this.sendMessageController.sendMessageToStreamer( - messageType, + 'XRButtonPressed', + data + ) + ); + this.streamMessageController.registerMessageHandler( + MessageDirection.ToStreamer, + 'XRButtonReleased', + (data: Array) => + this.sendMessageController.sendMessageToStreamer( + 'XRButtonReleased', + data + ) + ); + this.streamMessageController.registerMessageHandler( + MessageDirection.ToStreamer, + 'XRAnalog', + (data: Array) => + this.sendMessageController.sendMessageToStreamer( + 'XRAnalog', data ) ); @@ -807,15 +869,6 @@ export class WebRtcPlayerController { } } - /** - * Send an Iframe request to the streamer - */ - requestKeyFrame() { - this.streamMessageController.toStreamerHandlers.get('IFrameRequest')( - 'IFrameRequest' - ); - } - /** * Loads a freeze frame if it is required otherwise shows the play overlay */ @@ -996,9 +1049,12 @@ export class WebRtcPlayerController { signallingServerUrl += '?' + Flags.PreferSFU + '=true'; } - // If we are sending the offer add a special url parameter to the url, making sure we append correctly - if (this.config.isFlagEnabled(Flags.BrowserSendOffer)) { - signallingServerUrl += (signallingServerUrl.includes('?') ? '&' : '?') + Flags.BrowserSendOffer + '=true'; + // If we are sending the offer add a special url parameter to the url, making sure we append correctly + if (this.config.isFlagEnabled(Flags.BrowserSendOffer)) { + signallingServerUrl += + (signallingServerUrl.includes('?') ? '&' : '?') + + Flags.BrowserSendOffer + + '=true'; } return signallingServerUrl; @@ -1041,7 +1097,7 @@ export class WebRtcPlayerController { this.peerConnectionController = new PeerConnectionController( peerConfig, this.config, - this.preferredCodec + this.preferredCodec ); // set up peer connection controller video stats @@ -1157,9 +1213,16 @@ export class WebRtcPlayerController { * Handles when the signalling server gives us the list of streamer ids. */ handleStreamerListMessage(messageStreamerList: MessageStreamerList) { - Logger.Log(Logger.GetStackTrace(), `Got streamer list ${messageStreamerList.ids}`, 6); + Logger.Log( + Logger.GetStackTrace(), + `Got streamer list ${messageStreamerList.ids}`, + 6 + ); messageStreamerList.ids.unshift(''); // add an empty option at the top - this.config.setOptionSettingOptions(OptionParameters.StreamerId, messageStreamerList.ids); + this.config.setOptionSettingOptions( + OptionParameters.StreamerId, + messageStreamerList.ids + ); } /** @@ -1185,11 +1248,11 @@ export class WebRtcPlayerController { handleWebRtcOffer(Offer: MessageOffer) { Logger.Log(Logger.GetStackTrace(), `Got offer sdp ${Offer.sdp}`, 6); - this.isUsingSFU = (Offer.sfu) ? Offer.sfu : false; - if(this.isUsingSFU) { - // Disable negotiating with the sfu as the sfu only supports one codec at a time - this.peerConnectionController.preferredCodec = ""; - } + this.isUsingSFU = Offer.sfu ? Offer.sfu : false; + if (this.isUsingSFU) { + // Disable negotiating with the sfu as the sfu only supports one codec at a time + this.peerConnectionController.preferredCodec = ''; + } const sdpOffer: RTCSessionDescriptionInit = { sdp: Offer.sdp, @@ -1470,7 +1533,7 @@ export class WebRtcPlayerController { '---- Sending Request for an IFrame ----', 6 ); - this.streamMessageController.toStreamerHandlers.get('IFrameRequest'); + this.streamMessageController.toStreamerHandlers.get('IFrameRequest')(); } /** @@ -1648,12 +1711,12 @@ export class WebRtcPlayerController { setDisconnectMessageOverride(message: string): void { this.disconnectMessageOverride = message; } - - setPreferredCodec(codec: string) { - this.preferredCodec = codec; - if(this.peerConnectionController) { - this.peerConnectionController.preferredCodec = codec; - this.peerConnectionController.updateCodecSelection = false; - } - } + + setPreferredCodec(codec: string) { + this.preferredCodec = codec; + if (this.peerConnectionController) { + this.peerConnectionController.preferredCodec = codec; + this.peerConnectionController.updateCodecSelection = false; + } + } } diff --git a/Frontend/library/src/WebSockets/SignallingProtocol.ts b/Frontend/library/src/WebSockets/SignallingProtocol.ts index 667e4534..e9b7b866 100644 --- a/Frontend/library/src/WebSockets/SignallingProtocol.ts +++ b/Frontend/library/src/WebSockets/SignallingProtocol.ts @@ -81,8 +81,13 @@ export class SignallingProtocol { websocketController.signallingProtocol.addMessageHandler( MessageRecvTypes.STREAMER_LIST, (listPayload: string) => { - Logger.Log(Logger.GetStackTrace(), MessageRecvTypes.STREAMER_LIST, 6); - const streamerList: MessageStreamerList = JSON.parse(listPayload); + Logger.Log( + Logger.GetStackTrace(), + MessageRecvTypes.STREAMER_LIST, + 6 + ); + const streamerList: MessageStreamerList = + JSON.parse(listPayload); websocketController.onStreamerList(streamerList); } ); diff --git a/Frontend/library/src/WebXR/WebXRController.ts b/Frontend/library/src/WebXR/WebXRController.ts new file mode 100644 index 00000000..edd1473d --- /dev/null +++ b/Frontend/library/src/WebXR/WebXRController.ts @@ -0,0 +1,294 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +import { Logger } from '../Logger/Logger'; +import { WebRtcPlayerController } from '../pixelstreamingfrontend'; +import { WebGLUtils } from '../Util/WebGLUtils'; +import { Controller } from '../Inputs/GamepadTypes'; +import { XRGamepadController } from '../Inputs/XRGamepadController'; + +export class WebXRController { + xrSession: XRSession; + xrRefSpace: XRReferenceSpace; + gl: WebGL2RenderingContext; + + positionLocation: number; + texcoordLocation: number; + resolutionLocation: WebGLUniformLocation; + offsetLocation: WebGLUniformLocation; + + positionBuffer: WebGLBuffer; + texcoordBuffer: WebGLBuffer; + + webRtcController: WebRtcPlayerController; + xrGamepadController: XRGamepadController; + xrControllers: Array; + + constructor(webRtcPlayerController: WebRtcPlayerController) { + this.xrSession = null; + this.webRtcController = webRtcPlayerController; + this.xrControllers = []; + this.xrGamepadController = new XRGamepadController( + this.webRtcController.streamMessageController + ); + } + + public xrClicked() { + if (!this.xrSession) { + navigator.xr + .requestSession('immersive-vr') + .then((session: XRSession) => { + this.onXrSessionStarted(session); + }); + } else { + this.xrSession.end(); + } + } + + onXrSessionEnded() { + Logger.Log(Logger.GetStackTrace(), 'XR Session ended'); + this.xrSession = null; + } + + onXrSessionStarted(session: XRSession) { + Logger.Log(Logger.GetStackTrace(), 'XR Session started'); + + this.xrSession = session; + this.xrSession.addEventListener('end', () => { + this.onXrSessionEnded(); + }); + + const canvas = document.createElement('canvas'); + this.gl = canvas.getContext('webgl2', { + xrCompatible: true + }); + + this.xrSession.updateRenderState({ + baseLayer: new XRWebGLLayer(this.xrSession, this.gl) + }); + + // setup vertex shader + const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER); + this.gl.shaderSource(vertexShader, WebGLUtils.vertexShader()); + this.gl.compileShader(vertexShader); + + // setup fragment shader + const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER); + this.gl.shaderSource(fragmentShader, WebGLUtils.fragmentShader()); + this.gl.compileShader(fragmentShader); + + // setup GLSL program + const shaderProgram = this.gl.createProgram(); + this.gl.attachShader(shaderProgram, vertexShader); + this.gl.attachShader(shaderProgram, fragmentShader); + this.gl.linkProgram(shaderProgram); + this.gl.useProgram(shaderProgram); + + // look up where vertex data needs to go + this.positionLocation = this.gl.getAttribLocation( + shaderProgram, + 'a_position' + ); + this.texcoordLocation = this.gl.getAttribLocation( + shaderProgram, + 'a_texCoord' + ); + // Create a buffer to put three 2d clip space points in + this.positionBuffer = this.gl.createBuffer(); + // Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer) + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.positionBuffer); + + // Turn on the position attribute + this.gl.enableVertexAttribArray(this.positionLocation); + // Create a texture. + const texture = this.gl.createTexture(); + this.gl.bindTexture(this.gl.TEXTURE_2D, texture); + // Set the parameters so we can render any size image. + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_WRAP_S, + this.gl.CLAMP_TO_EDGE + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_WRAP_T, + this.gl.CLAMP_TO_EDGE + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_MIN_FILTER, + this.gl.NEAREST + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_MAG_FILTER, + this.gl.NEAREST + ); + + this.texcoordBuffer = this.gl.createBuffer(); + // lookup uniforms + this.resolutionLocation = this.gl.getUniformLocation( + shaderProgram, + 'u_resolution' + ); + this.offsetLocation = this.gl.getUniformLocation( + shaderProgram, + 'u_offset' + ); + + session.requestReferenceSpace('local').then((refSpace) => { + this.xrRefSpace = refSpace; + this.xrSession.requestAnimationFrame( + (time: DOMHighResTimeStamp, frame: XRFrame) => + this.onXrFrame(time, frame) + ); + }); + } + + onXrFrame(time: DOMHighResTimeStamp, frame: XRFrame) { + const pose = frame.getViewerPose(this.xrRefSpace); + if (pose) { + const matrix = pose.transform.matrix; + const mat = []; + for (let i = 0; i < 16; i++) { + mat[i] = new Float32Array([matrix[i]])[0]; + } + + // prettier-ignore + this.webRtcController.streamMessageController.toStreamerHandlers.get('XRHMDTransform')([ + mat[0], mat[4], mat[8], mat[12], + mat[1], mat[5], mat[9], mat[13], + mat[2], mat[6], mat[10], mat[14], + mat[3], mat[7], mat[11], mat[15] + ]); + + const glLayer = this.xrSession.renderState.baseLayer; + // If we do have a valid pose, bind the WebGL layer's framebuffer, + // which is where any content to be displayed on the XRDevice must be + // rendered. + this.gl.bindFramebuffer(this.gl.FRAMEBUFFER, glLayer.framebuffer); + + // Upload the image into the texture. WebGL knows how to extract the current frame from the video element + this.gl.texImage2D( + this.gl.TEXTURE_2D, + 0, + this.gl.RGBA, + this.gl.RGBA, + this.gl.UNSIGNED_BYTE, + this.webRtcController.videoPlayer.getVideoElement() + ); + this.render(this.webRtcController.videoPlayer.getVideoElement()); + } + + this.xrSession.inputSources.forEach( + (source: XRInputSource, index: number, array: XRInputSource[]) => { + this.xrGamepadController.updateStatus( + source, + frame, + this.xrRefSpace + ); + }, + this + ); + + this.xrSession.requestAnimationFrame( + (time: DOMHighResTimeStamp, frame: XRFrame) => + this.onXrFrame(time, frame) + ); + } + + private render(videoElement: HTMLVideoElement) { + if (!this.gl) { + return; + } + + const glLayer = this.xrSession.renderState.baseLayer; + this.gl.viewport( + 0, + 0, + glLayer.framebufferWidth, + glLayer.framebufferHeight + ); + this.gl.uniform4f(this.offsetLocation, 1.0, 1.0, 0.0, 0.0); + + // Set rectangle + // prettier-ignore + this.gl.bufferData( + this.gl.ARRAY_BUFFER, + new Float32Array([ + 0, 0, + videoElement.videoWidth, 0, + 0, videoElement.videoHeight, + 0, videoElement.videoHeight, + videoElement.videoWidth, 0, + videoElement.videoWidth, videoElement.videoHeight + ]), + this.gl.STATIC_DRAW + ); + + // Provide texture coordinates for the rectangle + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.texcoordBuffer); + this.gl.bufferData( + this.gl.ARRAY_BUFFER, + new Float32Array([ + 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0 + ]), + this.gl.STATIC_DRAW + ); + + let size; // components per iteration + let type; // the data type + let normalize; // normalize the data + let stride; // 0 = move forward size * sizeof(type) each iteration to get the next position + let offset; // start position of the buffer + + // Bind the position buffer. + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.positionBuffer); + // Tell the position attribute how to get data out of positionBuffer (ARRAY_BUFFER) + size = 2; // 2 components per iteration + type = this.gl.FLOAT; // the data is 32bit floats + normalize = false; // don't normalize the data + stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position + offset = 0; // start at the beginning of the buffer + this.gl.vertexAttribPointer( + this.positionLocation, + size, + type, + normalize, + stride, + offset + ); + // Turn on the texcoord attribute + this.gl.enableVertexAttribArray(this.texcoordLocation); + // bind the texcoord buffer. + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.texcoordBuffer); + // Tell the texcoord attribute how to get data out of texcoordBuffer (ARRAY_BUFFER) + size = 2; // 2 components per iteration + type = this.gl.FLOAT; // the data is 32bit floats + normalize = false; // don't normalize the data + stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position + offset = 0; // start at the beginning of the buffer + this.gl.vertexAttribPointer( + this.texcoordLocation, + size, + type, + normalize, + stride, + offset + ); + // set the resolution + this.gl.uniform2f( + this.resolutionLocation, + videoElement.videoWidth, + videoElement.videoHeight + ); + // draw the rectangle. + const primitiveType = this.gl.TRIANGLES; + const count = 6; + offset = 0; + this.gl.drawArrays(primitiveType, offset, count); + } + + static isSessionSupported(mode: XRSessionMode): Promise { + return navigator.xr.isSessionSupported(mode); + } +} diff --git a/Frontend/library/src/pixelstreamingfrontend.ts b/Frontend/library/src/pixelstreamingfrontend.ts index 071d99e5..926290e8 100644 --- a/Frontend/library/src/pixelstreamingfrontend.ts +++ b/Frontend/library/src/pixelstreamingfrontend.ts @@ -1,6 +1,7 @@ // Copyright Epic Games, Inc. All Rights Reserved. export { WebRtcPlayerController } from './WebRtcPlayer/WebRtcPlayerController'; +export { WebXRController } from './WebXr/WebXRController'; export { Config, ControlSchemeType,