Jellyfin Web VR plugin for Quest 3 (VR180 stereo playback)
Injects a 「VR 播放」 button into Jellyfin Web's video player page and plays
VR180 / VR360 stereo content on Meta Quest 3 directly in the browser, with
DirectPlay 8K HEVC, in-VR subtitles and an in-VR control panel.
Modules (src/, concatenated in filename order into dist/jfvr.js):
00-core.js namespace, shared state, GL helpers, layer/frame-hook registry
10-subtitles.js libass canvas -> head-locked quad layer, auto-cropped to ink,
with depth/size decoupling (angular-size lock)
20-controls.js in-VR control panel: progress/seek, transport, subtitle rows,
aiming beam, padlock, pure 2D canvas -> XR quad layer
30-hands.js system hand model (Three.js + generic-hand GLB) via a manually
driven projection layer
90-main.js boot, XR session lifecycle, rAF broker (rAF stops during XR),
input dispatch, settings panel, localStorage persistence
Key on-device findings are documented in HANDOFF.md §6 and README.md addenda:
requestAnimationFrame stops during an XR session, 3:2 pulldown at 90 Hz needs
updateTargetFrameRate(120), libass must be resized before entering VR, a quad's
local -Z is its plane normal, and hand aiming must use the gaze ray because the
emulated targetRaySpace points sideways. Quest does not expose eye tracking.
Docs: INTERFACES.md (module contract + addenda), README.md (measurements),
HANDOFF.md (handover notes). Credentials are NOT in the repo: deploy.py reads
the SSH password from the JF_PW environment variable.
This commit is contained in:
+343
@@ -0,0 +1,343 @@
|
||||
/*!
|
||||
* JFVR core — shared namespace, state, helpers, registries.
|
||||
* Loaded first. Everything else attaches to window.__JFVR.
|
||||
*
|
||||
* See INTERFACES.md for the binding contract.
|
||||
*/
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
if (window.__JFVR && window.__JFVR.__core) return; // idempotent
|
||||
|
||||
var NS = window.__JFVR = window.__JFVR || {};
|
||||
|
||||
NS.version = 'hud-5'; // hud-4 = ADDENDUM 7; hud-5 = ADDENDUM 7b (padlock in the panel + locked mode)
|
||||
NS.__core = true;
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* logging
|
||||
* ------------------------------------------------------------------ */
|
||||
NS.log = function () {
|
||||
var a = Array.prototype.slice.call(arguments);
|
||||
try {
|
||||
console.log.apply(console, ['%c[JFVR ' + NS.version + ']', 'color:#4ea1ff;font-weight:bold'].concat(a));
|
||||
} catch (e) {
|
||||
try { console.log('[JFVR]', a.join(' ')); } catch (e2) {}
|
||||
}
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* shared state (modules read/write; integrator owns lifecycle)
|
||||
* ------------------------------------------------------------------ */
|
||||
var S = NS.state = {
|
||||
video: null,
|
||||
session: null,
|
||||
refSpace: null,
|
||||
viewerSpace: null,
|
||||
|
||||
projection: '180', // '180' | '360'
|
||||
stereo: 'sbs', // 'mono' | 'sbs' | 'ou'
|
||||
|
||||
subtitles: true,
|
||||
ctlLock: false, // ADDENDUM 7: control-bar lock badge (per VR session)
|
||||
subMode: 'hud', // 'hud' | 'sphere' | 'off'
|
||||
subAlpha: 'premultiplied', // 'premultiplied' | 'straight'
|
||||
subOpacity: 0.85, // 0..1, applied to the subtitle layer live
|
||||
// y is the quad CENTRE; negative = below eye level. -0.42 keeps the strip
|
||||
// clearly under the visual centre rather than crowding it.
|
||||
subPlacement: { y: -0.42, z: -1.45, width: 2.0 },
|
||||
// ADDENDUM 6 — depth / size decoupling. With subLockAngle ON, changing the
|
||||
// strip's DEPTH holds its angular size (subAngleDeg, the strip's full
|
||||
// angular width in degrees) and its angular height (the y/z ratio), so the
|
||||
// user can pull the subtitles to the depth of a nearby object without the
|
||||
// text blowing up or dropping out of the lower view. 69.2 deg is exactly
|
||||
// the pre-ADDENDUM-6 look: 2.0 m wide at 1.45 m.
|
||||
subLockAngle: true,
|
||||
subAngleDeg: 69.2,
|
||||
|
||||
controls: true,
|
||||
ctlAutoHideMs: 4000, // in-VR control bar auto-hide
|
||||
|
||||
// written every XR frame by 90-main.js, read by modules
|
||||
pointer: null, // { visible, source, origin:{x,y,z}, dir:{x,y,z}, hit:{nx,ny}|null }
|
||||
|
||||
stats: {},
|
||||
errors: []
|
||||
};
|
||||
|
||||
// live handles (mirrors of state, convenient for modules)
|
||||
Object.defineProperty(NS, 'session', {
|
||||
get: function () { return S.session; },
|
||||
set: function (v) { S.session = v; }
|
||||
});
|
||||
Object.defineProperty(NS, 'video', {
|
||||
get: function () { return S.video; },
|
||||
set: function (v) { S.video = v; }
|
||||
});
|
||||
Object.defineProperty(NS, 'refSpace', {
|
||||
get: function () { return S.refSpace; },
|
||||
set: function (v) { S.refSpace = v; }
|
||||
});
|
||||
Object.defineProperty(NS, 'viewerSpace', {
|
||||
get: function () { return S.viewerSpace; },
|
||||
set: function (v) { S.viewerSpace = v; }
|
||||
});
|
||||
|
||||
NS.recordError = function (where, e) {
|
||||
var msg = where + ': ' + ((e && e.message) || e);
|
||||
S.errors.push(msg);
|
||||
if (S.errors.length > 40) S.errors.shift();
|
||||
return msg;
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* DOM helpers
|
||||
* ------------------------------------------------------------------ */
|
||||
NS.makeEl = function (tag, className, text) {
|
||||
var el = document.createElement(tag);
|
||||
if (className) el.className = className;
|
||||
if (text != null) el.textContent = text;
|
||||
return el;
|
||||
};
|
||||
|
||||
NS.clamp = function (v, lo, hi) { return v < lo ? lo : (v > hi ? hi : v); };
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* GL helpers
|
||||
* ------------------------------------------------------------------ */
|
||||
NS.compileShader = function (gl, type, src) {
|
||||
var sh = gl.createShader(type);
|
||||
gl.shaderSource(sh, src);
|
||||
gl.compileShader(sh);
|
||||
if (!gl.getShaderParameter(sh, gl.COMPILE_STATUS)) {
|
||||
var info = gl.getShaderInfoLog(sh) || 'shader compile failed';
|
||||
gl.deleteShader(sh);
|
||||
throw new Error(info);
|
||||
}
|
||||
return sh;
|
||||
};
|
||||
|
||||
// Fullscreen triangle in clip space. Fragment shader gets `in vec2 vUv`
|
||||
// with vUv.y already flipped so (0,0) is the top-left of the source image.
|
||||
NS.QUAD_VS = [
|
||||
'#version 300 es',
|
||||
'in vec2 aPos;',
|
||||
'out vec2 vUv;',
|
||||
'void main(){',
|
||||
' vUv = vec2(aPos.x * 0.5 + 0.5, 0.5 - aPos.y * 0.5);',
|
||||
' gl_Position = vec4(aPos, 0.0, 1.0);',
|
||||
'}'
|
||||
].join('\n');
|
||||
|
||||
NS.bindFullscreenTriangle = function (gl, program) {
|
||||
var buf = gl.createBuffer();
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, buf);
|
||||
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 3, -1, -1, 3]), gl.STATIC_DRAW);
|
||||
var loc = gl.getAttribLocation(program, 'aPos');
|
||||
gl.enableVertexAttribArray(loc);
|
||||
gl.vertexAttribPointer(loc, 2, gl.FLOAT, false, 0, 0);
|
||||
return loc;
|
||||
};
|
||||
|
||||
NS.linkProgram = function (gl, vsSrc, fsSrc) {
|
||||
var prog = gl.createProgram();
|
||||
gl.attachShader(prog, NS.compileShader(gl, gl.VERTEX_SHADER, vsSrc));
|
||||
gl.attachShader(prog, NS.compileShader(gl, gl.FRAGMENT_SHADER, fsSrc));
|
||||
gl.linkProgram(prog);
|
||||
if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) {
|
||||
var info = gl.getProgramInfoLog(prog) || 'program link failed';
|
||||
throw new Error(info);
|
||||
}
|
||||
gl.useProgram(prog);
|
||||
return prog;
|
||||
};
|
||||
|
||||
NS.makeTexture = function (gl) {
|
||||
var t = gl.createTexture();
|
||||
gl.bindTexture(gl.TEXTURE_2D, t);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
return t;
|
||||
};
|
||||
|
||||
NS.xrGl = function (alpha) {
|
||||
var c = document.createElement('canvas');
|
||||
return c.getContext('webgl2', {
|
||||
xrCompatible: true,
|
||||
alpha: alpha !== false,
|
||||
premultipliedAlpha: true,
|
||||
antialias: false,
|
||||
depth: false,
|
||||
desynchronized: true
|
||||
});
|
||||
};
|
||||
|
||||
// 4x4 matrix inverse (column-major, as used by WebXR).
|
||||
NS.invert4 = function (m) {
|
||||
var a00 = m[0], a01 = m[1], a02 = m[2], a03 = m[3],
|
||||
a10 = m[4], a11 = m[5], a12 = m[6], a13 = m[7],
|
||||
a20 = m[8], a21 = m[9], a22 = m[10], a23 = m[11],
|
||||
a30 = m[12], a31 = m[13], a32 = m[14], a33 = m[15],
|
||||
b00 = a00 * a11 - a01 * a10, b01 = a00 * a12 - a02 * a10,
|
||||
b02 = a00 * a13 - a03 * a10, b03 = a01 * a12 - a02 * a11,
|
||||
b04 = a01 * a13 - a03 * a11, b05 = a02 * a13 - a03 * a12,
|
||||
b06 = a20 * a31 - a21 * a30, b07 = a20 * a32 - a22 * a30,
|
||||
b08 = a20 * a33 - a23 * a30, b09 = a21 * a32 - a22 * a31,
|
||||
b10 = a21 * a33 - a23 * a31, b11 = a22 * a33 - a23 * a32,
|
||||
det = b00 * b11 - b01 * b10 + b02 * b09 + b03 * b08 - b04 * b07 + b05 * b06;
|
||||
if (!det) return new Float32Array(16);
|
||||
det = 1.0 / det;
|
||||
return new Float32Array([
|
||||
(a11 * b11 - a12 * b10 + a13 * b09) * det,
|
||||
(a02 * b10 - a01 * b11 - a03 * b09) * det,
|
||||
(a31 * b05 - a32 * b04 + a33 * b03) * det,
|
||||
(a22 * b04 - a21 * b05 - a23 * b03) * det,
|
||||
(a12 * b08 - a10 * b11 - a13 * b07) * det,
|
||||
(a00 * b11 - a02 * b08 + a03 * b07) * det,
|
||||
(a32 * b02 - a30 * b05 - a33 * b01) * det,
|
||||
(a20 * b05 - a22 * b02 + a23 * b01) * det,
|
||||
(a10 * b10 - a11 * b08 + a13 * b06) * det,
|
||||
(a01 * b08 - a00 * b10 - a03 * b06) * det,
|
||||
(a30 * b04 - a31 * b02 + a33 * b00) * det,
|
||||
(a21 * b02 - a20 * b04 - a23 * b00) * det,
|
||||
(a11 * b07 - a10 * b09 - a12 * b06) * det,
|
||||
(a00 * b09 - a01 * b07 + a02 * b06) * det,
|
||||
(a31 * b01 - a30 * b03 - a32 * b00) * det,
|
||||
(a20 * b03 - a21 * b01 + a22 * b00) * det
|
||||
]);
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* XR layer + frame-hook registry
|
||||
*
|
||||
* Layers are owned by modules; main.js owns pushing the render state.
|
||||
* ------------------------------------------------------------------ */
|
||||
var layers = [];
|
||||
var hooks = [];
|
||||
var pushRenderState = null; // set by main.js once a session exists
|
||||
|
||||
NS.setRenderStatePusher = function (fn) { pushRenderState = fn; };
|
||||
|
||||
NS.addLayer = function (layer) {
|
||||
if (!layer || layers.indexOf(layer) >= 0) return layer;
|
||||
layers.push(layer);
|
||||
NS.log('layer added (' + layers.length + ' total)');
|
||||
if (pushRenderState) { try { pushRenderState(); } catch (e) { NS.recordError('addLayer', e); } }
|
||||
return layer;
|
||||
};
|
||||
|
||||
NS.removeLayer = function (layer) {
|
||||
var i = layers.indexOf(layer);
|
||||
if (i < 0) return;
|
||||
layers.splice(i, 1);
|
||||
NS.log('layer removed (' + layers.length + ' left)');
|
||||
if (pushRenderState) { try { pushRenderState(); } catch (e) { NS.recordError('removeLayer', e); } }
|
||||
};
|
||||
|
||||
NS.getLayers = function () { return layers.slice(); };
|
||||
|
||||
NS.clearLayers = function () {
|
||||
layers.length = 0;
|
||||
if (pushRenderState) { try { pushRenderState(); } catch (e) {} }
|
||||
};
|
||||
|
||||
NS.addFrameHook = function (fn) {
|
||||
if (typeof fn !== 'function' || hooks.indexOf(fn) >= 0) return fn;
|
||||
hooks.push(fn);
|
||||
return fn;
|
||||
};
|
||||
|
||||
NS.removeFrameHook = function (fn) {
|
||||
var i = hooks.indexOf(fn);
|
||||
if (i >= 0) hooks.splice(i, 1);
|
||||
};
|
||||
|
||||
NS.runFrameHooks = function (frame, time, session) {
|
||||
for (var i = 0; i < hooks.length; i++) {
|
||||
try { hooks[i](frame, time, session); }
|
||||
catch (e) {
|
||||
var msg = NS.recordError('frameHook', e);
|
||||
if (!S.stats.hookErr) S.stats.hookErr = msg;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
NS.clearFrameHooks = function () { hooks.length = 0; };
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* stereo / projection helpers shared by modules
|
||||
* ------------------------------------------------------------------ */
|
||||
NS.stereoLayout = function () {
|
||||
return S.stereo === 'sbs' ? 'stereo-left-right'
|
||||
: S.stereo === 'ou' ? 'stereo-top-bottom'
|
||||
: 'mono';
|
||||
};
|
||||
|
||||
NS.centralAngle = function () {
|
||||
return S.projection === '360' ? Math.PI * 2 : Math.PI;
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* capabilities probe (safe at load time: no user gesture needed)
|
||||
* ------------------------------------------------------------------ */
|
||||
NS.caps = {
|
||||
ua: navigator.userAgent,
|
||||
secureContext: !!window.isSecureContext,
|
||||
hasXR: !!navigator.xr,
|
||||
immersiveVR: null,
|
||||
hasXRMediaBinding: typeof window.XRMediaBinding !== 'undefined',
|
||||
hasEquirectLayer: false,
|
||||
hasXRWebGLBinding: typeof window.XRWebGLBinding !== 'undefined',
|
||||
hasHandTracking: typeof window.XRHand !== 'undefined',
|
||||
hasXRJointSpace: typeof window.XRJointSpace !== 'undefined',
|
||||
hasGetJointPose: typeof window.XRFrame !== 'undefined' && typeof XRFrame.prototype.getJointPose === 'function',
|
||||
maxTextureSize: null,
|
||||
hevc: {},
|
||||
decoding8k: null
|
||||
};
|
||||
|
||||
(function probe() {
|
||||
try {
|
||||
var mb = window.XRMediaBinding;
|
||||
if (mb && mb.prototype) NS.caps.hasEquirectLayer = typeof mb.prototype.createEquirectLayer === 'function';
|
||||
var wg = window.XRWebGLBinding;
|
||||
if (wg && wg.prototype) NS.caps.hasQuadLayer = typeof wg.prototype.createQuadLayer === 'function';
|
||||
} catch (e) {}
|
||||
try {
|
||||
var gl = document.createElement('canvas').getContext('webgl2') ||
|
||||
document.createElement('canvas').getContext('webgl');
|
||||
if (gl) NS.caps.maxTextureSize = gl.getParameter(gl.MAX_TEXTURE_SIZE);
|
||||
} catch (e) {}
|
||||
try {
|
||||
['video/mp4; codecs="hev1.1.6.L183.B0"',
|
||||
'video/mp4; codecs="hvc1.1.6.L183.B0"',
|
||||
'video/mp4; codecs="hev1"',
|
||||
'video/mp4; codecs="hvc1"'].forEach(function (cs) {
|
||||
NS.caps.hevc[cs] = (typeof MediaSource !== 'undefined' && MediaSource.isTypeSupported(cs)) ||
|
||||
document.createElement('video').canPlayType(cs) || 'no';
|
||||
});
|
||||
} catch (e) {}
|
||||
if (navigator.xr && navigator.xr.isSessionSupported) {
|
||||
navigator.xr.isSessionSupported('immersive-vr')
|
||||
.then(function (v) { NS.caps.immersiveVR = v; })
|
||||
.catch(function (e) { NS.caps.immersiveVR = 'err:' + e.message; });
|
||||
}
|
||||
})();
|
||||
|
||||
NS.capsDiag = function () {
|
||||
var c = NS.caps;
|
||||
return [
|
||||
'origin : ' + location.origin,
|
||||
'secureContext : ' + c.secureContext,
|
||||
'immersive-vr : ' + c.immersiveVR,
|
||||
'XRMediaBinding : ' + c.hasXRMediaBinding,
|
||||
'XRWebGLBinding : ' + c.hasXRWebGLBinding,
|
||||
'createQuadLayer : ' + c.hasQuadLayer,
|
||||
'hand tracking API : ' + c.hasHandTracking + ' / jointSpace=' + c.hasXRJointSpace + ' / getJointPose=' + c.hasGetJointPose,
|
||||
'MAX_TEXTURE_SIZE : ' + c.maxTextureSize,
|
||||
'HEVC : ' + JSON.stringify(c.hevc)
|
||||
].join('\n');
|
||||
};
|
||||
})();
|
||||
+1022
File diff suppressed because it is too large
Load Diff
+1940
File diff suppressed because it is too large
Load Diff
+569
@@ -0,0 +1,569 @@
|
||||
/* 30-hands.js — __JFVR.Hands
|
||||
*
|
||||
* Draws the official Meta generic-hand models (the @webxr-input-profiles GLBs)
|
||||
* so the user sees their real hands in VR, posed from the 25 WebXR joint poses
|
||||
* the runtime reports. This replaces the hand-rolled laser beam / cursor.
|
||||
*
|
||||
* Three.js draws the models, but it does NOT own the XR session. This page
|
||||
* already composes several layers (an XRMediaBinding equirect video layer plus
|
||||
* quad layers for subtitles and the control panel); calling
|
||||
* renderer.xr.setSession() would install a baseLayer, which is illegal next to
|
||||
* `layers`. So instead:
|
||||
* - we build our own xrCompatible WebGL2 context,
|
||||
* - create a projection layer with XRWebGLBinding and RETURN it, so
|
||||
* 90-main.js can put it in updateRenderState({ layers }),
|
||||
* - drive renderer.render() by hand from a __JFVR frame hook, binding the
|
||||
* layer's swapchain framebuffer and each view's viewport ourselves.
|
||||
*
|
||||
* Interface (INTERFACES.md): init() / create(session) / destroy() / isReady() /
|
||||
* getLayer(). Nothing here reads document, the network or WebXR at load time;
|
||||
* only init()/create() do.
|
||||
*/
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
var NS = (typeof window !== 'undefined' && window.__JFVR) ? window.__JFVR : null;
|
||||
if (!NS || NS.Hands) return; // core missing, or already installed
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* constants
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
var CDN = 'https://cdn.jsdelivr.net/npm/';
|
||||
var THREE_VERSION = 'three@0.160.0';
|
||||
var THREE_URL = CDN + THREE_VERSION + '/build/three.module.js';
|
||||
var ADDONS_URL = CDN + THREE_VERSION + '/examples/jsm/';
|
||||
var IMPORT_MAP = {
|
||||
imports: {
|
||||
'three': THREE_URL,
|
||||
'three/addons/': ADDONS_URL
|
||||
}
|
||||
};
|
||||
|
||||
var HAND_URL = {
|
||||
left: CDN + '@webxr-input-profiles/assets@1.0/dist/profiles/generic-hand/left.glb',
|
||||
right: CDN + '@webxr-input-profiles/assets@1.0/dist/profiles/generic-hand/right.glb'
|
||||
};
|
||||
|
||||
// The 25 WebXR hand joints. The generic-hand GLB names its joint nodes after
|
||||
// these exact strings, so a normalised name lookup maps bone -> joint.
|
||||
var JOINT_NAMES = [
|
||||
'wrist',
|
||||
'thumb-metacarpal', 'thumb-phalanx-proximal', 'thumb-phalanx-distal', 'thumb-tip',
|
||||
'index-finger-metacarpal', 'index-finger-phalanx-proximal', 'index-finger-phalanx-intermediate', 'index-finger-phalanx-distal', 'index-finger-tip',
|
||||
'middle-finger-metacarpal', 'middle-finger-phalanx-proximal', 'middle-finger-phalanx-intermediate', 'middle-finger-phalanx-distal', 'middle-finger-tip',
|
||||
'ring-finger-metacarpal', 'ring-finger-phalanx-proximal', 'ring-finger-phalanx-intermediate', 'ring-finger-phalanx-distal', 'ring-finger-tip',
|
||||
'pinky-finger-metacarpal', 'pinky-finger-phalanx-proximal', 'pinky-finger-phalanx-intermediate', 'pinky-finger-phalanx-distal', 'pinky-finger-tip'
|
||||
];
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* state
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
var S = {
|
||||
// asset state: survives sessions, so a re-entry reuses the loaded GLBs
|
||||
inited: false,
|
||||
ready: false,
|
||||
THREE: null,
|
||||
models: { left: null, right: null },
|
||||
|
||||
// session state: torn down by destroy()
|
||||
session: null,
|
||||
gl: null,
|
||||
binding: null,
|
||||
projectionLayer: null,
|
||||
fbo: null,
|
||||
renderer: null,
|
||||
scene: null,
|
||||
camera: null,
|
||||
hookFn: null,
|
||||
frames: 0
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* helpers
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
function log() {
|
||||
try {
|
||||
if (NS && typeof NS.log === 'function') { NS.log.apply(NS, arguments); return; }
|
||||
} catch (e) { /* fall through */ }
|
||||
try { console.log.apply(console, ['[JFVR hands]'].concat(Array.prototype.slice.call(arguments))); } catch (e2) {}
|
||||
}
|
||||
|
||||
function msg(e) {
|
||||
return (e && e.message) ? e.message : String(e);
|
||||
}
|
||||
|
||||
function stats() {
|
||||
if (!NS.state) NS.state = {};
|
||||
if (!NS.state.stats) NS.state.stats = {};
|
||||
return NS.state.stats;
|
||||
}
|
||||
|
||||
function setStat(k, v) {
|
||||
try { stats()[k] = v; } catch (e) {}
|
||||
}
|
||||
|
||||
function setErr(text) {
|
||||
setStat('handsErr', text);
|
||||
}
|
||||
|
||||
function refSpaceOf() {
|
||||
var proj = S.projectionLayer;
|
||||
if (proj && proj.space) return proj.space; // layers with a space
|
||||
if (NS.refSpace) return NS.refSpace;
|
||||
return (NS.state && NS.state.refSpace) || null;
|
||||
}
|
||||
|
||||
function normalizeName(n) {
|
||||
return String(n == null ? '' : n).toLowerCase().replace(/_/g, '-');
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* asset loading (async, kicked off by init())
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
// 90-main.js has no import map, so Three's modules cannot resolve their bare
|
||||
// `three` import until we install one. Must happen before the first dynamic
|
||||
// import of a bare specifier, which is why init() does it up front.
|
||||
function ensureImportMap() {
|
||||
if (!document || !document.head) return;
|
||||
if (document.querySelector('script[type="importmap"]')) return;
|
||||
var im = document.createElement('script');
|
||||
im.type = 'importmap';
|
||||
im.textContent = JSON.stringify(IMPORT_MAP);
|
||||
document.head.appendChild(im);
|
||||
}
|
||||
|
||||
function buildModel(gltf, handedness) {
|
||||
var root = gltf && (gltf.scene || (gltf.scenes && gltf.scenes[0]));
|
||||
if (!root) { log('hands[' + handedness + ']: GLB has no scene'); return null; }
|
||||
|
||||
// The mesh is skinned: its bind-pose bounds are wrong once the joints move,
|
||||
// so frustum culling would make the hand pop in and out. Never cull it.
|
||||
root.traverse(function (o) {
|
||||
if (o.isMesh || o.isSkinnedMesh) o.frustumCulled = false;
|
||||
});
|
||||
|
||||
var byName = {};
|
||||
root.traverse(function (o) {
|
||||
if (o.name) byName[normalizeName(o.name)] = o;
|
||||
});
|
||||
|
||||
var bones = {};
|
||||
var missing = [];
|
||||
for (var i = 0; i < JOINT_NAMES.length; i++) {
|
||||
var joint = JOINT_NAMES[i];
|
||||
var bone = byName[normalizeName(joint)];
|
||||
if (bone) bones[joint] = bone;
|
||||
else missing.push(joint);
|
||||
}
|
||||
// Tolerate a renamed/partial skeleton: report once, pose what we can.
|
||||
if (missing.length) {
|
||||
log('hands[' + handedness + ']: no GLB bone for ' + missing.join(', ') + ' (joint skipped)');
|
||||
}
|
||||
return { root: root, bones: bones, missing: missing };
|
||||
}
|
||||
|
||||
function loadModels(THREE, GLTFLoader) {
|
||||
var loader = new GLTFLoader();
|
||||
function one(handedness) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
loader.load(HAND_URL[handedness], function (gltf) {
|
||||
var model = buildModel(gltf, handedness);
|
||||
if (!model) { reject(new Error(handedness + ': empty scene')); return; }
|
||||
S.models[handedness] = model;
|
||||
if (S.scene) S.scene.add(model.root); // layer may already be rendering
|
||||
resolve();
|
||||
}, undefined, function (e) {
|
||||
reject(new Error(handedness + ': ' + msg(e)));
|
||||
});
|
||||
});
|
||||
}
|
||||
return Promise.all([one('left'), one('right')]);
|
||||
}
|
||||
|
||||
function load() {
|
||||
return import('three').then(function (THREE) {
|
||||
S.THREE = THREE;
|
||||
return import('three/addons/loaders/GLTFLoader.js').then(function (mod) {
|
||||
return loadModels(THREE, mod.GLTFLoader);
|
||||
});
|
||||
}).then(function () {
|
||||
S.ready = true;
|
||||
setStat('handsReady', true);
|
||||
setErr(null);
|
||||
ensureRenderer(); // if create() already ran, start drawing the models
|
||||
log('hand models ready');
|
||||
}).catch(function (e) {
|
||||
setErr('load: ' + msg(e));
|
||||
log('hand model load failed:', msg(e));
|
||||
});
|
||||
}
|
||||
|
||||
function init() {
|
||||
if (S.inited) return;
|
||||
S.inited = true;
|
||||
setStat('handsReady', false);
|
||||
setStat('handsErr', null);
|
||||
setStat('handsSources', 'none');
|
||||
setStat('handsFrames', 0);
|
||||
try {
|
||||
ensureImportMap();
|
||||
var p = load();
|
||||
if (p && typeof p.catch === 'function') {
|
||||
p.catch(function (e) { setErr('init: ' + msg(e)); }); // belt and braces
|
||||
}
|
||||
} catch (e) {
|
||||
// Never propagate: a missing import map / document must not kill boot.
|
||||
setErr('init: ' + msg(e));
|
||||
log('hands init failed:', msg(e));
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* three.js + scene (built lazily, once THREE and a session are available)
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
function makeGl() {
|
||||
if (!document || !document.createElement) return null;
|
||||
var canvas = document.createElement('canvas');
|
||||
// A dedicated context: the other modules keep their own for the quad
|
||||
// layers, and the projection layer must render into its own swapchain.
|
||||
return canvas.getContext('webgl2', {
|
||||
xrCompatible: true,
|
||||
alpha: true,
|
||||
premultipliedAlpha: true,
|
||||
antialias: true,
|
||||
depth: true,
|
||||
stencil: false
|
||||
}) || null;
|
||||
}
|
||||
|
||||
function ensureRenderer() {
|
||||
if (S.renderer || !S.gl || !S.THREE) return S.renderer;
|
||||
try {
|
||||
var THREE = S.THREE;
|
||||
var renderer = new THREE.WebGLRenderer({
|
||||
canvas: S.gl.canvas,
|
||||
context: S.gl,
|
||||
alpha: true,
|
||||
antialias: true
|
||||
});
|
||||
// NEVER renderer.xr.setSession(): that sets a baseLayer and would clash
|
||||
// with the page's `layers` array. We bind the projection framebuffer and
|
||||
// render manually from the frame hook instead.
|
||||
renderer.autoClear = true;
|
||||
renderer.setClearColor(0x000000, 0); // transparent: the video shows through
|
||||
|
||||
var scene = new THREE.Scene();
|
||||
// The generic-hand GLBs use PBR materials; without a light they render
|
||||
// black, so give them a hemisphere fill plus a key light.
|
||||
scene.add(new THREE.HemisphereLight(0xffffff, 0x445566, 2.5));
|
||||
var key = new THREE.DirectionalLight(0xffffff, 1.4);
|
||||
key.position.set(0.3, 0.8, 0.5);
|
||||
scene.add(key);
|
||||
|
||||
var camera = new THREE.PerspectiveCamera();
|
||||
// The hook writes this camera's matrices straight from the XR view, so
|
||||
// stop three from recomputing matrixWorld from position/quaternion.
|
||||
camera.matrixAutoUpdate = false;
|
||||
camera.matrixWorldAutoUpdate = false;
|
||||
|
||||
S.renderer = renderer;
|
||||
S.scene = scene;
|
||||
S.camera = camera;
|
||||
['left', 'right'].forEach(function (h) {
|
||||
if (S.models[h]) scene.add(S.models[h].root);
|
||||
});
|
||||
log('three renderer ready for hands');
|
||||
} catch (e) {
|
||||
setErr('renderer: ' + msg(e));
|
||||
log('renderer setup failed:', msg(e));
|
||||
S.renderer = null;
|
||||
S.scene = null;
|
||||
S.camera = null;
|
||||
}
|
||||
return S.renderer;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* XR projection layer + manual render
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
// Resolve the framebuffer + viewport to render `view` into.
|
||||
//
|
||||
// Two shapes are accepted, in order:
|
||||
// 1. a projection layer that exposes `framebuffer` + `getViewport(view)`;
|
||||
// 2. the WebXR Layers spec path, XRWebGLBinding.getViewSubImage(layer, view),
|
||||
// which returns this view's viewport plus the swapchain colour/depth
|
||||
// textures; we attach those to a framebuffer of our own. (The spec gives
|
||||
// XRProjectionLayer no framebuffer and no getViewport(), so on a real
|
||||
// runtime branch 2 is the one that runs.)
|
||||
function bindViewTarget(view) {
|
||||
var gl = S.gl;
|
||||
var proj = S.projectionLayer;
|
||||
var binding = S.binding;
|
||||
if (!gl || !proj) return null;
|
||||
|
||||
if (proj.framebuffer && typeof proj.getViewport === 'function') {
|
||||
var direct = null;
|
||||
try { direct = proj.getViewport(view); } catch (e) { direct = null; }
|
||||
if (direct) {
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, proj.framebuffer);
|
||||
return { framebuffer: proj.framebuffer, viewport: direct };
|
||||
}
|
||||
}
|
||||
|
||||
if (binding && typeof binding.getViewSubImage === 'function') {
|
||||
var sub = null;
|
||||
try { sub = binding.getViewSubImage(proj, view); } catch (e) { sub = null; }
|
||||
if (sub && sub.colorTexture) {
|
||||
if (!S.fbo) S.fbo = gl.createFramebuffer();
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, S.fbo);
|
||||
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, sub.colorTexture, 0);
|
||||
if (sub.depthStencilTexture) {
|
||||
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.DEPTH_ATTACHMENT, gl.TEXTURE_2D, sub.depthStencilTexture, 0);
|
||||
}
|
||||
if (sub.viewport) return { framebuffer: S.fbo, viewport: sub.viewport };
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// Blank the layer while Three has not finished loading, so the compositor
|
||||
// does not show stale content.
|
||||
function clearLayer(pose) {
|
||||
var gl = S.gl;
|
||||
var target = bindViewTarget(pose.views[0]);
|
||||
if (!target) return;
|
||||
var vp = target.viewport;
|
||||
gl.viewport(Math.round(vp.x), Math.round(vp.y), Math.round(vp.width), Math.round(vp.height));
|
||||
gl.clearColor(0, 0, 0, 0);
|
||||
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
|
||||
}
|
||||
|
||||
function renderFrame(frame) {
|
||||
var space = refSpaceOf();
|
||||
if (!space) return;
|
||||
var pose = null;
|
||||
try { pose = frame.getViewerPose(space); } catch (e) { pose = null; }
|
||||
if (!pose || !pose.views || !pose.views.length) return;
|
||||
|
||||
var renderer = S.renderer || ensureRenderer();
|
||||
if (!renderer) { clearLayer(pose); return; }
|
||||
|
||||
var gl = S.gl;
|
||||
var camera = S.camera;
|
||||
|
||||
// The layer colour texture holds BOTH eyes in one framebuffer, so a bare
|
||||
// gl.clear() would erase the eye already drawn. autoClear stays on (it
|
||||
// clears to the transparent clear colour above) and the scissor box keeps
|
||||
// each clear + draw inside that eye's viewport.
|
||||
gl.enable(gl.SCISSOR_TEST);
|
||||
for (var i = 0; i < pose.views.length; i++) {
|
||||
var view = pose.views[i];
|
||||
var target = bindViewTarget(view);
|
||||
if (!target) break;
|
||||
var vp = target.viewport;
|
||||
var x = Math.round(vp.x), y = Math.round(vp.y);
|
||||
var w = Math.round(vp.width), h = Math.round(vp.height);
|
||||
if (w <= 0 || h <= 0) continue;
|
||||
|
||||
gl.viewport(x, y, w, h);
|
||||
gl.scissor(x, y, w, h);
|
||||
|
||||
camera.projectionMatrix.fromArray(view.projectionMatrix);
|
||||
camera.projectionMatrixInverse.copy(camera.projectionMatrix).invert();
|
||||
camera.matrixWorldInverse.fromArray(view.transform.inverse.matrix);
|
||||
camera.matrixWorld.copy(camera.matrixWorldInverse).invert();
|
||||
|
||||
renderer.render(S.scene, camera);
|
||||
}
|
||||
gl.disable(gl.SCISSOR_TEST);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* joint posing
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
function poseHand(frame, handedness, hand, space) {
|
||||
var model = S.models[handedness];
|
||||
if (!model || !model.bones || !space) return;
|
||||
|
||||
for (var i = 0; i < JOINT_NAMES.length; i++) {
|
||||
var joint = JOINT_NAMES[i];
|
||||
var bone = model.bones[joint];
|
||||
if (!bone) continue; // unmatched bone: logged once at load
|
||||
|
||||
var jointSpace = null;
|
||||
try { jointSpace = hand.get(joint); } catch (e) { jointSpace = null; }
|
||||
if (!jointSpace) continue;
|
||||
|
||||
var jp = null;
|
||||
try { jp = frame.getJointPose(jointSpace, space); } catch (e) { jp = null; }
|
||||
if (!jp || !jp.transform) continue;
|
||||
|
||||
// The generic-hand skeleton is FLAT: every joint node is a direct child
|
||||
// of the armature root, which sits at identity. A joint's pose in the
|
||||
// reference space is therefore also its local transform, so feeding the
|
||||
// matrix to the bone's TRS and letting three propagate is exact. The
|
||||
// model is metre-scaled (hand span ~0.2 m), so jp.radius is NOT applied:
|
||||
// the mesh already matches the joint scale.
|
||||
bone.matrix.fromArray(jp.transform.matrix);
|
||||
bone.matrix.decompose(bone.position, bone.quaternion, bone.scale);
|
||||
}
|
||||
}
|
||||
|
||||
function poseHands(frame, session) {
|
||||
var left = false, right = false;
|
||||
var sources = (session && session.inputSources) || null;
|
||||
var space = refSpaceOf();
|
||||
|
||||
if (sources) {
|
||||
for (var i = 0; i < sources.length; i++) {
|
||||
var src = sources[i];
|
||||
var hand = src && src.hand;
|
||||
if (!hand || hand.size !== 25) continue; // XRHand: exactly 25 joints
|
||||
var handedness = src.handedness;
|
||||
if (handedness !== 'left' && handedness !== 'right') continue;
|
||||
if (handedness === 'left') left = true; else right = true;
|
||||
poseHand(frame, handedness, hand, space);
|
||||
}
|
||||
}
|
||||
|
||||
// Show a hand only while its input source is present.
|
||||
if (S.models.left && S.models.left.root) S.models.left.root.visible = left;
|
||||
if (S.models.right && S.models.right.root) S.models.right.root.visible = right;
|
||||
|
||||
setStat('handsSources', (left || right)
|
||||
? (left ? 'left' : '') + (left && right ? ',' : '') + (right ? 'right' : '')
|
||||
: 'none');
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* frame hook
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
function hook(frame, time, session) {
|
||||
if (!S.projectionLayer || !frame) return;
|
||||
S.frames++;
|
||||
setStat('handsFrames', S.frames);
|
||||
poseHands(frame, session);
|
||||
renderFrame(frame);
|
||||
void time;
|
||||
}
|
||||
|
||||
function registerHook() {
|
||||
if (S.hookFn) return;
|
||||
S.hookFn = hook;
|
||||
if (typeof NS.addFrameHook === 'function') NS.addFrameHook(hook);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* lifecycle
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
function create(session) {
|
||||
try {
|
||||
if (S.projectionLayer && S.session === session) return S.projectionLayer; // idempotent
|
||||
if (!session) { log('hands.create: no session'); return null; }
|
||||
|
||||
destroy(); // drop anything left from a previous session
|
||||
|
||||
if (typeof XRWebGLBinding === 'undefined') {
|
||||
setErr('no XRWebGLBinding');
|
||||
log('hands.create: XRWebGLBinding unavailable');
|
||||
return null;
|
||||
}
|
||||
|
||||
var gl = makeGl();
|
||||
if (!gl) { setErr('no webgl2 context'); log('hands.create: no webgl2 context'); return null; }
|
||||
|
||||
var binding = new XRWebGLBinding(session, gl);
|
||||
var proj = binding.createProjectionLayer({
|
||||
textureType: 'texture',
|
||||
colorFormat: gl.RGBA8,
|
||||
depthFormat: gl.DEPTH_COMPONENT24,
|
||||
scaleFactor: 1,
|
||||
alpha: true
|
||||
});
|
||||
if (!proj) { setErr('createProjectionLayer returned null'); log('hands.create: no projection layer'); return null; }
|
||||
|
||||
S.session = session;
|
||||
S.gl = gl;
|
||||
S.binding = binding;
|
||||
S.projectionLayer = proj;
|
||||
S.fbo = null;
|
||||
S.frames = 0;
|
||||
|
||||
ensureRenderer(); // no-op until THREE has loaded; the hook retries
|
||||
registerHook();
|
||||
setErr(null);
|
||||
log('hands projection layer created (models ' + (S.ready ? 'ready' : 'loading') + ')');
|
||||
return proj;
|
||||
} catch (e) {
|
||||
setErr('create: ' + msg(e));
|
||||
log('hands.create failed:', msg(e));
|
||||
try { destroy(); } catch (e2) { void e2; }
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function destroy() {
|
||||
try {
|
||||
if (S.hookFn && typeof NS.removeFrameHook === 'function') NS.removeFrameHook(S.hookFn);
|
||||
} catch (e) { void e; }
|
||||
S.hookFn = null;
|
||||
|
||||
try {
|
||||
if (S.projectionLayer && typeof NS.removeLayer === 'function') NS.removeLayer(S.projectionLayer);
|
||||
} catch (e) { void e; }
|
||||
|
||||
try { if (S.gl && S.fbo) S.gl.deleteFramebuffer(S.fbo); } catch (e) { void e; }
|
||||
S.fbo = null;
|
||||
|
||||
var renderer = S.renderer;
|
||||
S.renderer = null;
|
||||
if (renderer) {
|
||||
// Drop the context with the session: browsers cap live WebGL contexts,
|
||||
// and the next create() builds a bright new one. Only our own resources
|
||||
// are touched - the loaded models are kept for reuse.
|
||||
try { if (typeof renderer.forceContextLoss === 'function') renderer.forceContextLoss(); } catch (e) { void e; }
|
||||
try { renderer.dispose(); } catch (e) { void e; }
|
||||
}
|
||||
|
||||
S.scene = null;
|
||||
S.camera = null;
|
||||
S.binding = null;
|
||||
S.projectionLayer = null;
|
||||
S.gl = null;
|
||||
S.session = null;
|
||||
S.frames = 0;
|
||||
setStat('handsFrames', 0);
|
||||
setStat('handsSources', 'none');
|
||||
}
|
||||
|
||||
function isReady() {
|
||||
// Models loaded AND a live projection layer + renderer: genuinely drawing.
|
||||
return !!(S.ready && S.renderer && S.projectionLayer);
|
||||
}
|
||||
|
||||
function getLayer() {
|
||||
return S.projectionLayer || null;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* public API
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
NS.Hands = {
|
||||
init: init,
|
||||
create: create,
|
||||
destroy: destroy,
|
||||
isReady: isReady,
|
||||
getLayer: getLayer
|
||||
};
|
||||
})();
|
||||
+1412
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user