This skill covers building immersive VR and AR experiences that run directly in web browsers using the WebXR Device API. WebXR enables XR content without app store downloads, using standard web technologies (JavaScript/TypeScript, WebGL/WebGPU, HTML). It covers the API surface, framework choices, performance techniques, and deployment patterns.

## Key Points

1. Use BufferGeometry, never Geometry (Three.js)
2. Merge static meshes (reduce draw calls)
3. Use instanced rendering for repeated objects
4. Compress textures (Basis Universal → KTX2)
5. Use glTF for assets (GPU-optimized format)
6. Implement LOD based on distance
7. Avoid garbage collection spikes:
- Pre-allocate vectors and matrices
- Use object pools for frequently created/destroyed objects
- Avoid creating closures in render loop
- Navigate to URL directly
- Or use Progressive Web App (PWA) for home screen shortcut

WebXR Development

Purpose

This skill covers building immersive VR and AR experiences that run directly in web browsers using the WebXR Device API. WebXR enables XR content without app store downloads, using standard web technologies (JavaScript/TypeScript, WebGL/WebGPU, HTML). It covers the API surface, framework choices, performance techniques, and deployment patterns.

WebXR API Overview

Session Types

// VR Session — Full immersion, replaces the real world
const vrSession = await navigator.xr.requestSession('immersive-vr', {
  requiredFeatures: ['local-floor'],
  optionalFeatures: ['bounded-floor', 'hand-tracking']
});

// AR Session — Overlay content on camera feed
const arSession = await navigator.xr.requestSession('immersive-ar', {
  requiredFeatures: ['hit-test', 'local-floor'],
  optionalFeatures: ['plane-detection', 'anchors', 'light-estimation']
});

// Inline Session — 3D content in a webpage (non-immersive)
const inlineSession = await navigator.xr.requestSession('inline');

Feature Detection

async function checkXRSupport() {
  if (!navigator.xr) {
    return { supported: false, reason: 'WebXR not available' };
  }

  const features = {
    vr: await navigator.xr.isSessionSupported('immersive-vr'),
    ar: await navigator.xr.isSessionSupported('immersive-ar'),
    inline: await navigator.xr.isSessionSupported('inline')
  };

  return { supported: true, features };
}

// Optional feature probing
const optionalFeatures = [
  'hand-tracking',
  'hit-test',
  'plane-detection',
  'mesh-detection',
  'anchors',
  'depth-sensing',
  'light-estimation',
  'camera-access',
  'layers'
];

Core Render Loop

// The WebXR render loop differs from standard requestAnimationFrame
function onXRFrame(time, frame) {
  const session = frame.session;

  // Request next frame first
  session.requestAnimationFrame(onXRFrame);

  // Get viewer pose relative to reference space
  const pose = frame.getViewerPose(referenceSpace);
  if (!pose) return; // Tracking lost

  const glLayer = session.renderState.baseLayer;
  gl.bindFramebuffer(gl.FRAMEBUFFER, glLayer.framebuffer);
  gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);

  // Render once per view (typically 2 for stereo VR)
  for (const view of pose.views) {
    const viewport = glLayer.getViewport(view);
    gl.viewport(viewport.x, viewport.y, viewport.width, viewport.height);

    // view.transform — camera position/orientation
    // view.projectionMatrix — projection for this eye
    renderScene(view);
  }
}

// Start the XR render loop
session.requestAnimationFrame(onXRFrame);

Input Handling

// Controller/hand input via XRInputSource
session.addEventListener('inputsourceschange', (event) => {
  for (const source of event.added) {
    console.log(`Input added: ${source.handedness} ${source.targetRayMode}`);
    // targetRayMode: 'gaze', 'tracked-pointer', 'screen', 'transient-pointer'
  }
});

// Select events (primary action — trigger press, hand pinch, screen tap)
session.addEventListener('select', (event) => {
  const source = event.inputSource;
  const frame = event.frame;
  const targetRayPose = frame.getPose(source.targetRaySpace, referenceSpace);

  if (targetRayPose) {
    const origin = targetRayPose.transform.position;
    const direction = targetRayPose.transform.orientation;
    performRaycast(origin, direction);
  }
});

// Squeeze events (secondary action — grip press, hand grab)
session.addEventListener('squeeze', (event) => {
  // Handle grab interaction
});

// Gamepad API for buttons/axes
function readGamepad(inputSource) {
  if (inputSource.gamepad) {
    const { buttons, axes } = inputSource.gamepad;
    const trigger = buttons[0]?.value;    // 0.0 - 1.0
    const grip = buttons[1]?.value;       // 0.0 - 1.0
    const thumbstickX = axes[2];          // -1.0 to 1.0
    const thumbstickY = axes[3];          // -1.0 to 1.0
  }
}

Hand Tracking

function processHandInput(frame, inputSource, referenceSpace) {
  if (!inputSource.hand) return;

  const hand = inputSource.hand;
  // hand.size === 25 joints

  const joints = [
    'wrist',
    'thumb-metacarpal', 'thumb-phalanx-proximal',
    'thumb-phalanx-distal', 'thumb-tip',
    'index-finger-metacarpal', 'index-finger-phalanx-proximal',
    'index-finger-phalanx-intermediate', 'index-finger-phalanx-distal',
    'index-finger-tip',
    // ... middle, ring, pinky follow same pattern
  ];

  for (const jointName of joints) {
    const joint = hand.get(jointName);
    if (joint) {
      const jointPose = frame.getJointPose(joint, referenceSpace);
      if (jointPose) {
        const pos = jointPose.transform.position;
        const radius = jointPose.radius; // Joint sphere radius
        updateJointVisualization(jointName, pos, radius);
      }
    }
  }

  // Pinch detection (thumb tip to index tip distance)
  const thumbTip = frame.getJointPose(hand.get('thumb-tip'), referenceSpace);
  const indexTip = frame.getJointPose(hand.get('index-finger-tip'), referenceSpace);
  if (thumbTip && indexTip) {
    const distance = calculateDistance(
      thumbTip.transform.position,
      indexTip.transform.position
    );
    const isPinching = distance < 0.02; // 2cm threshold
  }
}

Framework Approaches

Three.js + WebXR

Three.js is the most widely used WebXR framework:

import * as THREE from 'three';
import { VRButton } from 'three/addons/webxr/VRButton.js';
import { XRControllerModelFactory } from 'three/addons/webxr/XRControllerModelFactory.js';

// Setup
const renderer = new THREE.WebGLRenderer({ antialias: true });
renderer.xr.enabled = true;
document.body.appendChild(VRButton.createButton(renderer));

// Scene
const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera(70, width / height, 0.01, 100);

// Controllers
const controller1 = renderer.xr.getController(0);
const controller2 = renderer.xr.getController(1);
scene.add(controller1, controller2);

// Controller models (auto-matches hardware)
const modelFactory = new XRControllerModelFactory();
const grip1 = renderer.xr.getControllerGrip(0);
grip1.add(modelFactory.createControllerModel(grip1));
scene.add(grip1);

// Interaction
controller1.addEventListener('selectstart', onSelectStart);
controller1.addEventListener('selectend', onSelectEnd);

// Render loop (automatic in three.js XR mode)
renderer.setAnimationLoop((time, frame) => {
  // Update game logic
  renderer.render(scene, camera);
});

A-Frame (Declarative)

<!-- A-Frame: HTML-based VR scenes -->
<a-scene>
  <!-- Environment -->
  <a-sky color="#87CEEB"></a-sky>
  <a-plane position="0 0 0" rotation="-90 0 0"
           width="10" height="10" color="#4a7c59"></a-plane>

  <!-- Interactive objects -->
  <a-box position="0 1 -3" color="#4CC3D9"
         animation="property: rotation; to: 0 360 0; dur: 3000; loop: true"
         class="clickable"></a-box>

  <!-- Hands -->
  <a-entity id="leftHand" hand-tracking-controls="hand: left;"></a-entity>
  <a-entity id="rightHand" hand-tracking-controls="hand: right;"></a-entity>

  <!-- Laser pointer interaction -->
  <a-entity laser-controls="hand: right" raycaster="objects: .clickable"></a-entity>

  <!-- Camera rig for locomotion -->
  <a-entity id="rig" movement-controls>
    <a-camera></a-camera>
  </a-entity>
</a-scene>

Babylon.js

const engine = new BABYLON.Engine(canvas, true);
const scene = new BABYLON.Scene(engine);

// Enable WebXR
const xr = await scene.createDefaultXRExperienceAsync({
  floorMeshes: [ground],
  uiOptions: {
    sessionMode: 'immersive-vr'
  },
  optionalFeatures: true
});

// Hand tracking
const handTracking = xr.baseExperience.featuresManager.enableFeature(
  BABYLON.WebXRFeatureName.HAND_TRACKING,
  'latest',
  { xrInput: xr.input }
);

// Teleportation
const teleportation = xr.baseExperience.featuresManager.enableFeature(
  BABYLON.WebXRFeatureName.TELEPORTATION,
  'stable',
  {
    xrInput: xr.input,
    floorMeshes: [ground],
    snapPositions: [new BABYLON.Vector3(0, 0, -5)]
  }
);

AR-Specific Features

Hit Testing

// Request hit test source
const hitTestSource = await session.requestHitTestSource({
  space: viewerSpace
});

// In render loop
function onXRFrame(time, frame) {
  const hitResults = frame.getHitTestResults(hitTestSource);

  if (hitResults.length > 0) {
    const hit = hitResults[0];
    const hitPose = hit.getPose(referenceSpace);

    // Position reticle/preview at hit point
    reticle.visible = true;
    reticle.matrix.fromArray(hitPose.transform.matrix);

    // On user tap, place object at hit location
  } else {
    reticle.visible = false;
  }
}

Plane Detection

// Enable plane detection
const session = await navigator.xr.requestSession('immersive-ar', {
  requiredFeatures: ['plane-detection']
});

function onXRFrame(time, frame) {
  const detectedPlanes = frame.detectedPlanes;

  for (const plane of detectedPlanes) {
    const planePose = frame.getPose(plane.planeSpace, referenceSpace);
    const polygon = plane.polygon; // Array of DOMPointReadOnly (boundary)
    const orientation = plane.orientation; // 'horizontal' or 'vertical'

    // Visualize or use plane for placement
    updatePlaneVisualization(plane, planePose, polygon);
  }
}

Light Estimation

const lightProbe = await session.requestLightProbe();

function onXRFrame(time, frame) {
  const estimate = frame.getLightEstimate(lightProbe);

  if (estimate) {
    // Spherical harmonics for ambient lighting
    const sh = estimate.sphericalHarmonicsCoefficients;
    // 9 sets of RGB coefficients (27 floats total)
    updateEnvironmentLighting(sh);

    // Primary light direction and intensity
    const direction = estimate.primaryLightDirection;
    const intensity = estimate.primaryLightIntensity;
    updateDirectionalLight(direction, intensity);
  }
}

Performance Optimization

WebXR-Specific Performance

Performance Budget (mobile browser XR):
├── JavaScript execution: < 4ms per frame
├── WebGL draw calls: < 50-100
├── Triangle count: < 100K visible
├── Texture memory: < 128MB
├── JS heap: < 256MB
└── Target: 72 Hz (13.9ms total frame time)

Optimization Strategies:
1. Use BufferGeometry, never Geometry (Three.js)
2. Merge static meshes (reduce draw calls)
3. Use instanced rendering for repeated objects
4. Compress textures (Basis Universal → KTX2)
5. Use glTF for assets (GPU-optimized format)
6. Implement LOD based on distance
7. Avoid garbage collection spikes:
   - Pre-allocate vectors and matrices
   - Use object pools for frequently created/destroyed objects
   - Avoid creating closures in render loop

Asset Loading

// Progressive loading pattern for WebXR
async function loadScene() {
  // 1. Load critical assets first (what user sees immediately)
  const [skybox, floor] = await Promise.all([
    loadTexture('skybox.ktx2'),
    loadModel('floor.glb')
  ]);
  scene.add(floor);

  // 2. Enter XR session (user can start immediately)
  await enterXR();

  // 3. Stream remaining assets in background
  const assetManifest = [
    { url: 'furniture.glb', priority: 1 },
    { url: 'decorations.glb', priority: 2 },
    { url: 'audio/ambient.mp3', priority: 3 }
  ];

  for (const asset of assetManifest.sort((a, b) => a.priority - b.priority)) {
    const loaded = await loadAsset(asset.url);
    scene.add(loaded);
    // Each asset appears as it loads — no loading screen needed
  }
}

WebXR Layers

// WebXR Layers improve rendering quality and performance
const session = await navigator.xr.requestSession('immersive-vr', {
  requiredFeatures: ['layers']
});

// Projection layer (main 3D scene) — replaces baseLayer
const projectionLayer = new XRWebGLLayer(session, gl);

// Quad layer (sharp 2D content, like text or video)
const quadLayer = new XRQuadLayer(session, {
  space: referenceSpace,
  viewPixelWidth: 1024,
  viewPixelHeight: 1024,
  transform: new XRRigidTransform({ x: 0, y: 1.5, z: -2 })
});
// Quad layers are composited by the runtime — sharper than rendering
// text into the 3D scene and sampling through lens distortion

session.updateRenderState({
  layers: [projectionLayer, quadLayer]
});

Deployment and Hosting

HTTPS Requirement

WebXR requires HTTPS (or localhost for development).

Development options:
├── localhost      — Works without HTTPS
├── ngrok/tunnels  — HTTPS tunnel to local server
├── Self-signed    — Must be accepted in browser
└── Let's Encrypt  — Free HTTPS for production

Meta Quest Browser:
- Navigate to URL directly
- Or use Progressive Web App (PWA) for home screen shortcut
- chrome://inspect for remote debugging

PWA for XR

// manifest.json for XR PWA
{
  "name": "My XR Experience",
  "short_name": "XR App",
  "start_url": "/",
  "display": "standalone",
  "orientation": "any",
  "icons": [
    { "src": "icon-192.png", "sizes": "192x192", "type": "image/png" },
    { "src": "icon-512.png", "sizes": "512x512", "type": "image/png" }
  ],
  "xr": {
    "immersive-vr": true,
    "immersive-ar": true
  }
}

Browser Compatibility

WebXR Support Matrix (as of 2025):
┌────────────────────┬────┬────┬──────┬────────┐
│ Browser            │ VR │ AR │ Hands│ Layers │
├────────────────────┼────┼────┼──────┼────────┤
│ Chrome (Android)   │ ~  │ Y  │ N    │ N      │
│ Chrome (Desktop)   │ Y  │ N  │ N    │ Y      │
│ Meta Quest Browser │ Y  │ Y  │ Y    │ Y      │
│ Firefox Reality    │ Y  │ ~  │ N    │ N      │
│ Safari (visionOS)  │ Y  │ ~  │ Y    │ Y      │
│ Edge (Desktop)     │ Y  │ N  │ N    │ Y      │
│ Pico Browser       │ Y  │ ~  │ Y    │ ~      │
└────────────────────┴────┴────┴──────┴────────┘
Y = Supported, ~ = Partial, N = Not supported

Use feature detection, not browser detection.
Always provide a fallback 2D experience.

When to Apply This Skill

Use this skill when:

• Building XR experiences that must work without app installation
• Creating AR product visualization or try-on experiences
• Prototyping VR interactions rapidly in the browser
• Building cross-platform XR with a single codebase
• Deploying XR content via URL sharing
• Integrating XR features into existing web applications