Skip to content
This repository has been archived by the owner on Mar 1, 2024. It is now read-only.

Commit

Permalink
Add experimental support for WebXR based experiences (#85)
Browse files Browse the repository at this point in the history 
* Add WebGL program. Refactor input to reduce arguments when calling handlers.

* Add XR gamepad handling. Make use of input API more consistent.

* Add support for rendering the video stream into the headset.

* Add handling for float data type when sending message to streamer.

* Run prettier pass on repo.
  • Loading branch information
@Belchy06
Belchy06 committed Feb 10, 2023
1 parent 84d26f1 commit 18ba9db
Show file tree
Hide file tree
Showing 30 changed files with 1,371 additions and 647 deletions.
11 changes: 11 additions & 0 deletions Frontend/library/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Frontend/library/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
"webpack-cli": "^5.0.1"
},
"dependencies": {
"@types/webxr": "^0.5.1",
"jss": "^10.9.2",
"jss-plugin-camel-case": "^10.9.2",
"jss-plugin-global": "^10.9.2",
Expand Down
31 changes: 21 additions & 10 deletions Frontend/library/src/Application/Application.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,14 @@ import { PlayOverlay } from '../Overlay/PlayOverlay';
import { InfoOverlay } from '../Overlay/InfoOverlay';
import { ErrorOverlay } from '../Overlay/ErrorOverlay';
import { MessageOnScreenKeyboard } from '../WebSockets/MessageReceive';
import { WebXRController } from '../WebXR/WebXRController';

/**
* Provides common base functionality for applications that extend this application
*/
export class Application {
public webRtcController: WebRtcPlayerController;
public webXrController: WebXRController;
public config: Config;

_rootElement: HTMLElement;
Expand Down Expand Up @@ -101,6 +103,8 @@ export class Application {
this.onScreenKeyboardHelper.showOnScreenKeyboard(command);

this.updateColors(this.config.isFlagEnabled(Flags.LightMode));

this.webXrController = new WebXRController(this.webRtcController);
}

public createOverlays(): void {
Expand Down Expand Up @@ -129,6 +133,10 @@ export class Application {
this.settingsPanel.settingsCloseButton.onclick = () =>
this.settingsClicked();

// Add WebXR button to controls
controls.xrIcon.rootElement.onclick = () =>
this.webXrController.xrClicked();

// setup the stats/info button
controls.statsIcon.rootElement.onclick = () => this.statsClicked();

Expand All @@ -155,7 +163,7 @@ export class Application {
'Request'
);
requestKeyframeButton.addOnClickListener(() => {
this.webRtcController.requestKeyFrame();
this.webRtcController.sendIframeRequest();
});

const commandsSectionElem = this.config.buildSectionWithHeading(
Expand Down Expand Up @@ -394,14 +402,14 @@ export class Application {
}
);

this.config.addOnOptionSettingChangedListener(
OptionParameters.PreferredCodec,
(newValue: string) => {
if(this.webRtcController) {
this.webRtcController.setPreferredCodec(newValue);
}
}
);
this.config.addOnOptionSettingChangedListener(
OptionParameters.PreferredCodec,
(newValue: string) => {
if (this.webRtcController) {
this.webRtcController.setPreferredCodec(newValue);
}
}
);
}

/**
Expand Down Expand Up @@ -547,7 +555,10 @@ export class Application {
setWebRtcPlayerController(webRtcPlayerController: WebRtcPlayerController) {
this.webRtcController = webRtcPlayerController;

this.webRtcController.setPreferredCodec(this.config.getSettingOption(OptionParameters.PreferredCodec).selected);
this.webRtcController.setPreferredCodec(
this.config.getSettingOption(OptionParameters.PreferredCodec)
.selected
);
this.webRtcController.resizePlayerStyle();

this.disconnectOverlay.onAction(() => {
Expand Down
151 changes: 83 additions & 68 deletions Frontend/library/src/Config/Config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ export class TextParameters {
*
*/
export class OptionParameters {
static PreferredCodec = 'PreferredCodec';
static PreferredCodec = 'PreferredCodec';
static StreamerId = 'StreamerId';
}

Expand All @@ -69,8 +69,8 @@ export class Config {
/* A map of text settings - e.g. signalling server url */
private textParameters = new Map<string, SettingText>();

/* A map of enum based settings - e.g. preferred codec */
private optionParameters = new Map<string, SettingOption>();
/* A map of enum based settings - e.g. preferred codec */
private optionParameters = new Map<string, SettingOption>();

// ------------ Settings -----------------

Expand Down Expand Up @@ -116,52 +116,61 @@ export class Config {
'Signalling url',
'Url of the signalling server',
(location.protocol === 'https:' ? 'wss://' : 'ws://') +
window.location.hostname +
// for readability, we omit the port if it's 80
((window.location.port === '80' || window.location.port === '') ? '' : `:${window.location.port}`)
window.location.hostname +
// for readability, we omit the port if it's 80
(window.location.port === '80' ||
window.location.port === ''
? ''
: `:${window.location.port}`)
)
);

this.optionParameters.set(OptionParameters.StreamerId,
this.optionParameters.set(
OptionParameters.StreamerId,
new SettingOption(
OptionParameters.StreamerId,
'Streamer ID',
'The ID of the streamer to stream.',
'',
[])
);

/**
* Enum Parameters
*/
this.optionParameters.set(OptionParameters.PreferredCodec,
new SettingOption(
OptionParameters.PreferredCodec,
'Preferred Codec',
'The preferred codec to be used during codec negotiation',
'H264 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
(function(): Array<string> {
const browserSupportedCodecs: Array<string> = [];
// Try get the info needed from the RTCRtpReceiver. This is only available on chrome
if(!RTCRtpReceiver.getCapabilities)
{
browserSupportedCodecs.push("Only available on Chrome");
return browserSupportedCodecs;
}

const matcher = /(VP\d|H26\d|AV1).*/
const codecs = RTCRtpReceiver.getCapabilities('video').codecs;
codecs.forEach(codec => {
const str = codec.mimeType.split("/")[1] + ' ' + (codec.sdpFmtpLine || '');
const match = matcher.exec(str);
if(match !== null) {
browserSupportedCodecs.push(str);
}
});
return browserSupportedCodecs;
})()
)
);
[]
)
);

/**
* Enum Parameters
*/
this.optionParameters.set(
OptionParameters.PreferredCodec,
new SettingOption(
OptionParameters.PreferredCodec,
'Preferred Codec',
'The preferred codec to be used during codec negotiation',
'H264 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
(function (): Array<string> {
const browserSupportedCodecs: Array<string> = [];
// Try get the info needed from the RTCRtpReceiver. This is only available on chrome
if (!RTCRtpReceiver.getCapabilities) {
browserSupportedCodecs.push('Only available on Chrome');
return browserSupportedCodecs;
}

const matcher = /(VP\d|H26\d|AV1).*/;
const codecs =
RTCRtpReceiver.getCapabilities('video').codecs;
codecs.forEach((codec) => {
const str =
codec.mimeType.split('/')[1] +
' ' +
(codec.sdpFmtpLine || '');
const match = matcher.exec(str);
if (match !== null) {
browserSupportedCodecs.push(str);
}
});
return browserSupportedCodecs;
})()
)
);

/**
* Boolean parameters
Expand Down Expand Up @@ -454,14 +463,17 @@ export class Config {
this.numericParameters.get(NumericParameters.AFKTimeoutSecs)
);

const preferredCodecOption = this.optionParameters.get(OptionParameters.PreferredCodec);
this.addSettingOption(
psSettingsSection,
preferredCodecOption
);
if([...preferredCodecOption.selector.options].map(o => o.value).includes("Only available on Chrome")) {
preferredCodecOption.disable();
}
const preferredCodecOption = this.optionParameters.get(
OptionParameters.PreferredCodec
);
this.addSettingOption(psSettingsSection, preferredCodecOption);
if (
[...preferredCodecOption.selector.options]
.map((o) => o.value)
.includes('Only available on Chrome')
) {
preferredCodecOption.disable();
}

/* Setup all view/ui related settings under this section */
const viewSettingsSection = this.buildSectionWithHeading(
Expand Down Expand Up @@ -536,14 +548,16 @@ export class Config {
}
}

addOnOptionSettingChangedListener(
id: string,
addOnOptionSettingChangedListener(
id: string,
onChangedListener: (newValue: string) => void
): void {
if(this.optionParameters.has(id)) {
this.optionParameters.get(id).addOnChangedListener(onChangedListener);
}
}
): void {
if (this.optionParameters.has(id)) {
this.optionParameters
.get(id)
.addOnChangedListener(onChangedListener);
}
}

/**
* @param id The id of the numeric setting we are interested in getting a value for.
Expand Down Expand Up @@ -649,21 +663,22 @@ export class Config {
this.numericParameters.set(setting.id, setting);
}

/**
/**
* Add an enum based settings element to a particular settings section in the DOM and registers that flag in the Config.enumParameters map.
* @param settingsSection The settings section HTML element.
* @param settingFlag The settings flag object.
*/
addSettingOption(settingsSection: HTMLElement,
setting: SettingOption
): void {
settingsSection.appendChild(setting.rootElement);
this.optionParameters.set(setting.id, setting);
}
addSettingOption(
settingsSection: HTMLElement,
setting: SettingOption
): void {
settingsSection.appendChild(setting.rootElement);
this.optionParameters.set(setting.id, setting);
}

getSettingOption(id: string): SettingOption {
return this.optionParameters.get(id);
}
getSettingOption(id: string): SettingOption {
return this.optionParameters.get(id);
}

/**
* Get the value of the configuration flag which has the given id.
Expand Down Expand Up @@ -706,7 +721,7 @@ export class Config {
}
}

/**
/**
* Set the option setting list of options.
* @param id The id of the setting
* @param settingOptions The values the setting could take
Expand All @@ -722,7 +737,7 @@ export class Config {
}
}

/**
/**
* Set option enum settings selected option.
* @param id The id of the setting
* @param settingOptions The value to select out of all the options
Expand Down
Loading

0 comments on commit 18ba9db

Please sign in to comment.