Initial commit: Unity Needle AR Menu project with MenuScene and SampleScene web apps
Add root .gitignore for Unity Library/Temp/Logs, IDE folders, and node_modules. Include Assets, Needle TypeScript (MenuController, asset picker, WebXR), and project configuration. Made-with: Cursor
This commit is contained in:
pelpanagiotis
618
.agents/skills/needle-engine/references/api.md
Normal file
618
.agents/skills/needle-engine/references/api.md
Normal file
@@ -0,0 +1,618 @@
|
||||
# Needle Engine — Core API Reference
|
||||
|
||||
## Table of Contents
|
||||
- [Lifecycle Methods](#lifecycle-methods-complete)
|
||||
- [Decorators](#decorators)
|
||||
- [Context API](#context-api-thiscontext)
|
||||
- [GameObject Utilities](#gameobject-utilities)
|
||||
- [Finding Objects](#finding-objects)
|
||||
- [Coroutines](#coroutines)
|
||||
- [Asset Loading at Runtime](#asset-loading-at-runtime)
|
||||
- [Renderer and Materials](#renderer-and-materials)
|
||||
- [Object3D Extensions](#object3d-extensions)
|
||||
- [Utilities](#utilities)
|
||||
- [Vite Plugin Options](#vite-plugin-options)
|
||||
|
||||
---
|
||||
|
||||
## Lifecycle Methods (complete)
|
||||
|
||||
All methods are optional — only implement what you need.
|
||||
|
||||
```ts
|
||||
class MyComponent extends Behaviour {
|
||||
// Initialization
|
||||
awake() // first, before Start, even if disabled
|
||||
onEnable() // whenever component/GO becomes active
|
||||
start() // once, on first enabled frame
|
||||
|
||||
// Per-frame
|
||||
earlyUpdate() // every frame, before update()
|
||||
update() // every frame
|
||||
lateUpdate() // every frame, after all update() runs
|
||||
onBeforeRender(frame: XRFrame | null) // just before Three.js renders
|
||||
onAfterRender() // just after Three.js renders
|
||||
|
||||
// Deactivation / cleanup
|
||||
onDisable() // when component/GO becomes inactive
|
||||
onDestroy() // called by destroy(obj) — NOT by removeComponent()
|
||||
|
||||
// Pointer events (requires an EventSystem + Raycaster in the scene)
|
||||
onPointerEnter?(args: PointerEventData) // pointer enters this object
|
||||
onPointerMove?(args: PointerEventData) // pointer moves over this object
|
||||
onPointerExit?(args: PointerEventData) // pointer leaves this object
|
||||
onPointerDown?(args: PointerEventData) // pointer button pressed on this object
|
||||
onPointerUp?(args: PointerEventData) // pointer button released
|
||||
onPointerClick?(args: PointerEventData) // full click on this object
|
||||
|
||||
// XR events
|
||||
supportsXR?(mode: XRSessionMode): boolean // filter which XR modes this component handles
|
||||
onBeforeXR?(mode: XRSessionMode, args: XRSessionInit) // modify session init params
|
||||
onEnterXR?(args: NeedleXREventArgs) // joined an XR session
|
||||
onUpdateXR?(args: NeedleXREventArgs) // per-frame during XR
|
||||
onLeaveXR?(args: NeedleXREventArgs) // left the XR session
|
||||
onXRControllerAdded?(args: NeedleXRControllerEventArgs) // controller connected
|
||||
onXRControllerRemoved?(args: NeedleXRControllerEventArgs) // controller disconnected
|
||||
|
||||
// Physics (requires Needle Collider component on same GameObject)
|
||||
onCollisionEnter(col: Collision)
|
||||
onCollisionStay(col: Collision)
|
||||
onCollisionExit(col: Collision)
|
||||
onTriggerEnter(col: Collision)
|
||||
onTriggerStay(col: Collision)
|
||||
onTriggerExit(col: Collision)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Decorators
|
||||
|
||||
| Decorator | Purpose |
|
||||
|---|---|
|
||||
| `@registerType` | Required on every component — registers the class for GLB deserialization |
|
||||
| `@serializable()` | Serialize/deserialize a primitive (number, string, boolean) |
|
||||
| `@serializable(Type)` | Serialize/deserialize a typed field (Object3D, Texture, Color, etc.) |
|
||||
| `@syncField()` | Auto-sync field over the network in a SyncedRoom |
|
||||
| `@syncField(onChange)` | Sync + call a callback when value changes remotely |
|
||||
|
||||
|
||||
### Serializable Types
|
||||
|
||||
```ts
|
||||
// Primitives — no type argument needed
|
||||
@serializable() myNumber!: number;
|
||||
@serializable() myString!: string;
|
||||
@serializable() myBool!: boolean;
|
||||
|
||||
// Complex types — pass the constructor
|
||||
import { RGBAColor, AssetReference } from "@needle-tools/engine";
|
||||
import { Object3D, Texture, Vector2, Vector3, Color } from "three";
|
||||
|
||||
@serializable(Object3D) myRef!: Object3D;
|
||||
@serializable(Texture) tex!: Texture;
|
||||
@serializable(RGBAColor) col!: RGBAColor;
|
||||
@serializable(AssetReference) asset!: AssetReference;
|
||||
@serializable(Vector3) pos!: Vector3;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Context API (`this.context`)
|
||||
|
||||
```ts
|
||||
this.context.scene // THREE.Scene
|
||||
this.context.mainCamera // THREE.Camera (currently active)
|
||||
this.context.renderer // THREE.WebGLRenderer
|
||||
this.context.domElement // <needle-engine> HTML element
|
||||
|
||||
// Time
|
||||
this.context.time.frame // frame counter (number)
|
||||
this.context.time.deltaTime // seconds since last frame (affected by timeScale)
|
||||
this.context.time.time // total elapsed seconds
|
||||
this.context.time.realtimeSinceStartup
|
||||
this.context.time.timeScale // default 1; affects deltaTime, animation, and audio
|
||||
|
||||
// Input — polling API (check in update())
|
||||
this.context.input.getPointerDown(index) // pointer just pressed this frame
|
||||
this.context.input.getPointerUp(index) // pointer just released this frame
|
||||
this.context.input.getPointerPressed(index) // pointer currently held
|
||||
this.context.input.getPointerPosition(index) // {x, y} in screen pixels
|
||||
this.context.input.getPointerPositionDelta(index) // movement since last frame
|
||||
this.context.input.getPointerPressedCount() // how many pointers are pressed
|
||||
this.context.input.mousePosition // shortcut for pointer 0 position
|
||||
this.context.input.getKeyDown(key) // "Space", "ArrowLeft", "a", etc.
|
||||
this.context.input.getKeyUp(key)
|
||||
this.context.input.getKeyPressed(key)
|
||||
|
||||
// Input — event-based API (subscribe/unsubscribe)
|
||||
this.context.input.addEventListener("pointerdown", (evt) => { /* NEPointerEvent */ });
|
||||
this.context.input.addEventListener("pointerup", callback);
|
||||
this.context.input.addEventListener("pointermove", callback);
|
||||
this.context.input.addEventListener("keydown", callback);
|
||||
this.context.input.removeEventListener("pointerdown", callback);
|
||||
|
||||
// Component pointer callbacks (require EventSystem + Raycaster in the scene):
|
||||
// onPointerEnter, onPointerMove, onPointerExit, onPointerDown, onPointerUp, onPointerClick
|
||||
// These fire on the specific object the pointer interacts with (see Lifecycle Methods)
|
||||
|
||||
// Physics — two raycast systems for different purposes:
|
||||
// 1. Visual raycast: hits rendered geometry (no collider needed)
|
||||
// Automatically builds MeshBVH (three-mesh-bvh) on web workers — falls back to standard
|
||||
// three.js raycasting until BVH is ready. Works with procedural geometry too.
|
||||
// Use for: UI interaction, picking visible objects, click detection
|
||||
|
||||
// Simplest usage — uses current pointer position, works in pointer event handlers:
|
||||
const hits = this.context.physics.raycast();
|
||||
|
||||
// With options:
|
||||
this.context.physics.raycast({ maxDistance: 100, layerMask: 0xff, ignore: [this.gameObject] })
|
||||
|
||||
// From a specific pixel position (e.g. in a raw pointerdown handler):
|
||||
// IMPORTANT: screenPoint is in normalized device coordinates (-1 to 1), NOT pixels!
|
||||
const hits = this.context.physics.raycast({
|
||||
screenPoint: new Vector2(
|
||||
(e.clientX / window.innerWidth) * 2 - 1,
|
||||
-(e.clientY / window.innerHeight) * 2 + 1
|
||||
),
|
||||
});
|
||||
|
||||
// DO NOT pass raw pixel coords as screenPoint — this is wrong:
|
||||
// ctx.physics.raycast({ screenPoint: new Vector2(e.clientX, e.clientY) }) // WRONG!
|
||||
|
||||
// 2. Physics engine raycast: hits Rapier colliders only
|
||||
// Use for: ground detection, line-of-sight, physics-based queries
|
||||
this.context.physics.engine?.raycast(origin, direction, { maxDistance, solid })
|
||||
this.context.physics.engine?.raycastAndGetNormal(origin, direction)
|
||||
this.context.physics.engine?.sphereOverlap(position, radius)
|
||||
|
||||
// Access Rapier world directly for advanced queries:
|
||||
this.context.physics.engine.world // underlying Rapier world
|
||||
|
||||
// Network
|
||||
this.context.connection // core networking manager (usable directly or via SyncedRoom)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## GameObject Utilities
|
||||
|
||||
```ts
|
||||
import { instantiate, destroy, GameObject } from "@needle-tools/engine";
|
||||
|
||||
// Component access
|
||||
go.getComponent(Type)
|
||||
go.getComponentInChildren(Type)
|
||||
go.getComponentInParent(Type)
|
||||
go.getComponents(Type) // all matching on same GO
|
||||
go.getComponentsInChildren(Type)
|
||||
|
||||
// Lifecycle
|
||||
instantiate(source, opts?) // preferred — clone; opts: { position, rotation, parent }
|
||||
destroy(obj) // destroys GO + calls onDestroy on components
|
||||
obj.removeComponent(comp) // removes without calling onDestroy
|
||||
|
||||
// Active state
|
||||
go.visible = false // hides in scene (still ticks)
|
||||
GameObject.setActive(go, false) // disables lifecycle callbacks
|
||||
|
||||
// Hierarchy
|
||||
go.contains(otherObj) // true if otherObj is a descendant (Needle extension on Object3D)
|
||||
|
||||
// World-space properties (Needle extensions on Object3D)
|
||||
go.worldPosition // get/set world position (Vector3)
|
||||
go.worldQuaternion // get/set world rotation (Quaternion)
|
||||
go.worldScale // get/set world scale (Vector3)
|
||||
go.worldForward // forward direction in world space (Vector3)
|
||||
go.worldRight // right direction in world space (Vector3)
|
||||
go.worldUp // up direction in world space (Vector3)
|
||||
|
||||
// Tag / name
|
||||
go.name // string
|
||||
go.userData.tags // string[] (set from Unity via Tag component)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Finding Objects
|
||||
|
||||
```ts
|
||||
import { findObjectOfType, findObjectsOfType } from "@needle-tools/engine";
|
||||
|
||||
findObjectOfType(MyComponent, ctx) // first match in scene
|
||||
findObjectsOfType(MyComponent, ctx) // all matches
|
||||
ctx.scene.getObjectByName("Player") // by name (Three.js built-in)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Coroutines
|
||||
|
||||
Generator functions that can yield across frames:
|
||||
|
||||
```ts
|
||||
import { WaitForSeconds, WaitForFrames, delayForFrames } from "@needle-tools/engine";
|
||||
|
||||
start() {
|
||||
this.startCoroutine(this.flashLight());
|
||||
}
|
||||
|
||||
*flashLight() {
|
||||
while (true) {
|
||||
this.light.visible = !this.light.visible;
|
||||
yield WaitForSeconds(0.5); // wait 0.5 seconds
|
||||
// yield; // wait exactly one frame
|
||||
// yield WaitForFrames(10); // wait N frames
|
||||
}
|
||||
}
|
||||
|
||||
// Stop all coroutines on this component:
|
||||
this.stopAllCoroutines();
|
||||
|
||||
// Async alternative (returns a Promise):
|
||||
await delayForFrames(5);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Asset Loading at Runtime
|
||||
|
||||
```ts
|
||||
import { AssetReference } from "@needle-tools/engine";
|
||||
|
||||
// Declare in component (set in Unity Inspector)
|
||||
@serializable(AssetReference) prefab!: AssetReference;
|
||||
|
||||
async start() {
|
||||
// Load and instantiate
|
||||
const instance = await this.prefab.instantiate({ parent: this.gameObject });
|
||||
|
||||
// Or just load the GLTF (no instantiate)
|
||||
const gltf = await this.prefab.loadAssetAsync();
|
||||
}
|
||||
```
|
||||
|
||||
Load a GLB by URL at runtime:
|
||||
```ts
|
||||
import { AssetReference } from "@needle-tools/engine";
|
||||
|
||||
const ref = AssetReference.getOrCreate(this.context.domElement.baseURI, "assets/extra.glb");
|
||||
const instance = await ref.instantiate({ parent: this.gameObject });
|
||||
```
|
||||
|
||||
Load any asset directly (without AssetReference):
|
||||
```ts
|
||||
import { loadAsset } from "@needle-tools/engine";
|
||||
|
||||
const model = await loadAsset("assets/model.glb");
|
||||
const obj = model.scene; // ← Object3D is on .scene, not the return value itself
|
||||
obj.traverse(n => { /* ... */ });
|
||||
```
|
||||
|
||||
> **`loadAsset()` returns a model wrapper** (with `.scene`, `.animations`, etc.) — not an Object3D directly. The wrapper type is universal regardless of format (GLB, FBX, OBJ, USDZ). Use `model.scene` to get the root Object3D.
|
||||
|
||||
> **Caching:** `AssetReference.getOrCreate()` caches by URL and returns the **same Object3D** on repeated calls. Adding a cached object to the scene again just moves it. Use `.instantiate()` for independent copies.
|
||||
|
||||
> **Note:** Needle Engine automatically handles KTX, Draco, and meshopt decompression — no loader setup needed.
|
||||
|
||||
---
|
||||
|
||||
## Renderer and Materials
|
||||
|
||||
### Accessing meshes and materials
|
||||
The `Renderer` component wraps Three.js meshes/materials. It's present on objects exported from Unity/Blender, but not automatically created for code-only objects — add it manually with `addComponent(Renderer)`, or access materials directly via Three.js (`(obj as Mesh).material`).
|
||||
|
||||
```ts
|
||||
import { Renderer } from "@needle-tools/engine";
|
||||
|
||||
const renderer = this.gameObject.getComponent(Renderer);
|
||||
|
||||
// Materials
|
||||
renderer.sharedMaterial // first material (read/write)
|
||||
renderer.sharedMaterials // all materials (array, index-assignable)
|
||||
renderer.sharedMaterials[0] = mat; // replace a material by index
|
||||
|
||||
// Meshes
|
||||
renderer.sharedMesh // first Mesh/SkinnedMesh Object3D
|
||||
renderer.sharedMeshes // all mesh Object3Ds (for multi-material groups)
|
||||
|
||||
// GPU Instancing — draws identical meshes in a single draw call for performance
|
||||
// In Unity/Blender: enable on the material or via the Needle UI on the object
|
||||
// In code:
|
||||
Renderer.setInstanced(obj, true); // enable instancing (also creates Renderer if missing)
|
||||
|
||||
// Visibility (without affecting hierarchy or component state)
|
||||
Renderer.setVisible(obj, false);
|
||||
```
|
||||
|
||||
### MaterialPropertyBlock — per-object material overrides
|
||||
Overrides material properties (color, texture, roughness, etc.) on a **per-object** basis without creating new material instances. Multiple objects can share the same material but look different. Overrides are applied in `onBeforeRender` and restored in `onAfterRender`.
|
||||
|
||||
```ts
|
||||
import { MaterialPropertyBlock } from "@needle-tools/engine";
|
||||
import { Color, Texture } from "three";
|
||||
|
||||
// Get or create a property block for an object (never use the constructor directly)
|
||||
const block = MaterialPropertyBlock.get(myMesh);
|
||||
|
||||
// Override properties
|
||||
block.setOverride("color", new Color(1, 0, 0));
|
||||
block.setOverride("roughness", 0.2);
|
||||
block.setOverride("map", myTexture);
|
||||
|
||||
// Override with UV transform (e.g. for lightmaps)
|
||||
block.setOverride("lightMap", lightmapTex, {
|
||||
offset: new Vector2(0.5, 0.5),
|
||||
repeat: new Vector2(2, 2)
|
||||
});
|
||||
|
||||
// Read back
|
||||
const color = block.getOverride("color")?.value;
|
||||
|
||||
// Remove overrides
|
||||
block.removeOveride("color"); // remove one
|
||||
block.clearAllOverrides(); // remove all
|
||||
block.dispose(); // remove the entire property block
|
||||
|
||||
// Check if an object has overrides
|
||||
MaterialPropertyBlock.hasOverrides(myMesh);
|
||||
```
|
||||
|
||||
Overrides are registered on the **Object3D**, not on the material — if you swap the material, overrides still apply to the new one. Use `dispose()` or `clearAllOverrides()` to remove them.
|
||||
|
||||
Common use cases: per-object colors/tinting, lightmaps, reflection probes, see-through/x-ray effects.
|
||||
|
||||
---
|
||||
|
||||
## Object3D Extensions
|
||||
|
||||
Needle Engine patches Three.js `Object3D.prototype` with convenience properties. These work on **any** Object3D in the scene.
|
||||
|
||||
### World transforms (getter + setter)
|
||||
```ts
|
||||
// GET — returns a temporary Vector3/Quaternion (don't store references, copy if needed)
|
||||
obj.worldPosition // Vector3 — world-space position
|
||||
obj.worldQuaternion // Quaternion — world-space rotation
|
||||
obj.worldRotation // Vector3 — world-space euler (degrees)
|
||||
obj.worldScale // Vector3 — world-space scale
|
||||
|
||||
// SET — must assign to apply (mutating the returned vector won't update the transform)
|
||||
obj.worldPosition = new Vector3(1, 2, 3); // sets world position
|
||||
obj.worldQuaternion = myQuat; // sets world rotation
|
||||
obj.worldScale = new Vector3(2, 2, 2); // sets world scale
|
||||
|
||||
// Direction vectors (read-only)
|
||||
obj.worldForward // Vector3 — forward direction in world space (0,0,1 rotated)
|
||||
obj.worldRight // Vector3 — right direction
|
||||
obj.worldUp // Vector3 — up direction
|
||||
|
||||
// worldForward also has a setter — point an object in a direction:
|
||||
obj.worldForward = targetDirection;
|
||||
```
|
||||
|
||||
The getters return **temporary vectors** from an internal pool — they're overwritten on the next call. You can read and re-assign them directly (`obj.worldPosition = other.worldPosition`). For temporary math use `getTempVector()`. Only use `.clone()` when you must store a value across frames — never in per-frame code.
|
||||
|
||||
### Component access
|
||||
```ts
|
||||
obj.getComponent(MyComponent) // first component of type
|
||||
obj.getComponentInChildren(MyComponent) // search children recursively
|
||||
obj.getComponentInParent(MyComponent) // search parents recursively
|
||||
obj.getComponents(MyComponent) // all of type on this object
|
||||
obj.getComponentsInChildren(MyComponent)
|
||||
obj.addComponent(MyComponent) // add a new component
|
||||
```
|
||||
|
||||
### Other extensions
|
||||
```ts
|
||||
obj.guid // get/set — unique identifier for networking (string | undefined)
|
||||
obj.contains(otherObj) // true if otherObj is a descendant
|
||||
obj.activeSelf // get/set active state (same as GameObject.setActive)
|
||||
```
|
||||
|
||||
`guid` is used by the networking system to identify objects across clients. Objects exported from Unity/Blender have guids automatically. For runtime-created objects, set `obj.guid = "my-id"` if they need to participate in networking (e.g. `syncInstantiate`, `SyncedTransform`).
|
||||
|
||||
### Bounding box and fitting
|
||||
```ts
|
||||
import { getBoundingBox, fitObjectIntoVolume } from "@needle-tools/engine";
|
||||
|
||||
// Get the bounding box of one or more objects
|
||||
const box = getBoundingBox(myObject); // single object
|
||||
const box = getBoundingBox([obj1, obj2, obj3]); // multiple objects
|
||||
const box = getBoundingBox(myObject, [ignoreThisChild]); // with objects to ignore
|
||||
const box = getBoundingBox(myObject, undefined, camera.layers); // filter by layer
|
||||
|
||||
const size = box.getSize(new Vector3());
|
||||
const center = box.getCenter(new Vector3());
|
||||
|
||||
// Fit an object into a target volume (scale + position)
|
||||
fitObjectIntoVolume(myObject, targetVolume);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Async Modules (`NEEDLE_ENGINE_MODULES`)
|
||||
|
||||
Heavy dependencies (physics, postprocessing, etc.) are loaded on demand, not bundled into the main entry point. **You do NOT need to call these for normal usage** — physics and postprocessing initialize automatically when their components are used. These are for advanced use cases like accessing the raw Rapier API or pmndrs postprocessing module directly.
|
||||
|
||||
```ts
|
||||
import { NEEDLE_ENGINE_MODULES } from "@needle-tools/engine";
|
||||
|
||||
// Available modules:
|
||||
NEEDLE_ENGINE_MODULES.RAPIER_PHYSICS // Rapier physics (WASM)
|
||||
NEEDLE_ENGINE_MODULES.POSTPROCESSING // pmndrs postprocessing
|
||||
NEEDLE_ENGINE_MODULES.POSTPROCESSING_AO // N8AO ambient occlusion
|
||||
NEEDLE_ENGINE_MODULES.MaterialX // MaterialX materials (WASM)
|
||||
NEEDLE_ENGINE_MODULES.PEERJS // PeerJS for networking
|
||||
|
||||
// Each module has:
|
||||
await module.load(); // trigger load + wait for it
|
||||
await module.ready(); // wait for load (doesn't trigger one)
|
||||
module.MODULE // the loaded module (undefined until loaded)
|
||||
module.MAYBEMODULE // null until loaded, then same as MODULE
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Utilities
|
||||
|
||||
All imported from `@needle-tools/engine`.
|
||||
|
||||
### Math (`Mathf`)
|
||||
```ts
|
||||
import { Mathf } from "@needle-tools/engine";
|
||||
|
||||
Mathf.lerp(a, b, t) // linear interpolation
|
||||
Mathf.clamp(value, min, max) // clamp to range
|
||||
Mathf.clamp01(value) // clamp to [0, 1]
|
||||
Mathf.remap(value, inMin, inMax, outMin, outMax) // remap between ranges
|
||||
Mathf.moveTowards(current, target, step) // step toward target
|
||||
Mathf.inverseLerp(a, b, value) // find t given value
|
||||
Mathf.toDegrees(radians)
|
||||
Mathf.toRadians(degrees)
|
||||
Mathf.random(min, max) // random in range (or random from array)
|
||||
Mathf.easeInOutCubic(t) // easing function
|
||||
```
|
||||
|
||||
### Temporary objects (avoid per-frame allocations)
|
||||
```ts
|
||||
import { getTempVector, getTempQuaternion } from "@needle-tools/engine";
|
||||
|
||||
// Returns reusable objects from a circular buffer — no GC pressure
|
||||
const v = getTempVector(1, 0, 0); // temporary Vector3
|
||||
const q = getTempQuaternion(); // temporary Quaternion
|
||||
// Don't store references — they get reused. Clone if you need to keep them.
|
||||
```
|
||||
|
||||
### Device detection (`DeviceUtilities`)
|
||||
```ts
|
||||
import { DeviceUtilities } from "@needle-tools/engine";
|
||||
|
||||
DeviceUtilities.isDesktop() // Windows/Mac (not headsets)
|
||||
DeviceUtilities.isMobileDevice() // phone or tablet
|
||||
DeviceUtilities.isiOS() // iPhone, iPad, Vision Pro
|
||||
DeviceUtilities.isAndroidDevice()
|
||||
DeviceUtilities.isQuest() // Meta Quest
|
||||
DeviceUtilities.isVisionOS() // Apple Vision Pro
|
||||
DeviceUtilities.isSafari()
|
||||
DeviceUtilities.supportsQuickLookAR() // USDZ/QuickLook support
|
||||
```
|
||||
|
||||
### Timing and delays
|
||||
```ts
|
||||
import { delay, delayForFrames, WaitForSeconds, WaitForFrames, WaitForPromise } from "@needle-tools/engine";
|
||||
|
||||
// Async
|
||||
await delay(1000); // wait 1 second
|
||||
await delayForFrames(5); // wait 5 frames
|
||||
|
||||
// In coroutines
|
||||
yield WaitForSeconds(0.5);
|
||||
yield WaitForFrames(10);
|
||||
yield WaitForPromise(fetch("/api")); // wait for a promise to resolve
|
||||
```
|
||||
|
||||
### User interaction
|
||||
```ts
|
||||
import { awaitInputAsync } from "@needle-tools/engine";
|
||||
|
||||
// Wait for the first user interaction (useful for audio autoplay policy)
|
||||
await awaitInputAsync();
|
||||
audioSource.play();
|
||||
```
|
||||
|
||||
### URL parameters
|
||||
```ts
|
||||
import { getParam, setParamWithoutReload } from "@needle-tools/engine";
|
||||
|
||||
const room = getParam("room"); // read ?room=xyz from URL
|
||||
setParamWithoutReload("room", "my-room"); // update URL without page reload
|
||||
```
|
||||
|
||||
### Debug messages (on-screen balloon)
|
||||
```ts
|
||||
import { showBalloonMessage, showBalloonWarning, showBalloonError } from "@needle-tools/engine";
|
||||
|
||||
showBalloonMessage("Hello!"); // info message on screen
|
||||
showBalloonWarning("Watch out!"); // warning (yellow)
|
||||
showBalloonError("Something broke!"); // error (red)
|
||||
```
|
||||
|
||||
### Debug console
|
||||
Append `?console` to the URL to show an on-screen debug console (uses vConsole). Useful for debugging on mobile devices where dev tools aren't available.
|
||||
|
||||
### Screenshots
|
||||
```ts
|
||||
import { screenshot2, saveImage } from "@needle-tools/engine";
|
||||
|
||||
// Simple screenshot (returns data URL)
|
||||
const dataUrl = screenshot2({ width: 1920, height: 1080 });
|
||||
saveImage(dataUrl, "screenshot.png");
|
||||
|
||||
// Screenshot as texture (apply to a material)
|
||||
const tex = screenshot2({ type: "texture", width: 512, height: 512 });
|
||||
|
||||
// Screenshot as blob
|
||||
const blob = await screenshot2({ type: "blob" });
|
||||
|
||||
// Share via Web Share API
|
||||
await screenshot2({ type: "share", title: "My Scene" });
|
||||
|
||||
// Transparent background
|
||||
screenshot2({ transparent: true, trim: true });
|
||||
|
||||
// XR screenshot (composites 3D scene over camera feed — requires "camera-access" feature)
|
||||
// Works in AR sessions when camera-access has been requested via onBeforeXR
|
||||
const xrScreenshot = screenshot2({ width: 1080, height: 1920 });
|
||||
```
|
||||
|
||||
### QR Code
|
||||
```ts
|
||||
import { generateQRCode } from "@needle-tools/engine";
|
||||
const qr = generateQRCode({ text: "https://mysite.com" });
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Vite Plugin Options
|
||||
|
||||
The `needlePlugins` function accepts user settings as the third argument. These control build behavior, optimization, and features.
|
||||
|
||||
```ts
|
||||
import { defineConfig } from "vite";
|
||||
import { needlePlugins } from "@needle-tools/engine/vite";
|
||||
|
||||
export default defineConfig(async ({ command }) => ({
|
||||
plugins: [
|
||||
...(await needlePlugins(command, {}, {
|
||||
// Key options:
|
||||
|
||||
// Make all external CDN URLs local for offline/self-contained deployments
|
||||
makeFilesLocal: true, // download everything
|
||||
// or: makeFilesLocal: "auto", // auto-detect which features to include
|
||||
// or: makeFilesLocal: { enabled: true, features: ["draco", "ktx2"] },
|
||||
|
||||
// PWA support (also install vite-plugin-pwa)
|
||||
pwa: true, // enable with defaults
|
||||
// or: pwa: { /* VitePWAOptions */ },
|
||||
|
||||
// Physics engine — set to false to tree-shake Rapier and reduce bundle size
|
||||
useRapier: false,
|
||||
|
||||
// Build pipeline — compression and optimization of glTF files
|
||||
noBuildPipeline: false, // default: runs optimization
|
||||
buildPipeline: {
|
||||
accessToken: process.env.NEEDLE_CLOUD_TOKEN, // use Needle Cloud for compression
|
||||
},
|
||||
|
||||
// Other options:
|
||||
// noAsap: true, // disable glTF preload links
|
||||
// noPoster: true, // disable poster image generation
|
||||
// openBrowser: true, // auto-open browser on local network IP
|
||||
})),
|
||||
],
|
||||
}));
|
||||
```
|
||||
|
||||
### `makeFilesLocal` features
|
||||
Downloads external CDN URLs at build time for fully self-contained deployments. Available features: `draco`, `ktx2`, `materialx`, `xr`, `skybox`, `fonts`, `needle-fonts`, `needle-models`, `needle-avatars`, `polyhaven`, `cdn-scripts`, `github-content`, `threejs-models`, `needle-uploads`.
|
||||
|
||||
449
.agents/skills/needle-engine/references/components.md
Normal file
449
.agents/skills/needle-engine/references/components.md
Normal file
@@ -0,0 +1,449 @@
|
||||
# Needle Engine — Built-in Components Reference
|
||||
|
||||
## Table of Contents
|
||||
- [Physics](#physics)
|
||||
- [Animation](#animation)
|
||||
- [Audio](#audio)
|
||||
- [Video](#video)
|
||||
- [Lighting and Shadows](#lighting-and-shadows)
|
||||
- [Post-Processing](#post-processing)
|
||||
- [Camera](#camera)
|
||||
- [Scene Switching](#scene-switching)
|
||||
- [Interaction](#interaction)
|
||||
- [Splines](#splines)
|
||||
- [Debug Tools](#debug-tools)
|
||||
- [Utilities](#utilities)
|
||||
|
||||
---
|
||||
|
||||
## Physics
|
||||
|
||||
See [physics.md](physics.md) for the full physics reference (colliders, Rigidbody API, raycasting, async loading).
|
||||
|
||||
Rapier initializes automatically — just add collider and rigidbody components. Use `SphereCollider` for balls, `CapsuleCollider` for characters/cylinders, not BoxCollider for everything. Use `applyImpulse` for one-shot actions, `applyForce` for continuous. Never access `rb._body` internals.
|
||||
|
||||
---
|
||||
|
||||
## Animation
|
||||
|
||||
### Animation (simple clip playback)
|
||||
```ts
|
||||
import { Animation } from "@needle-tools/engine";
|
||||
|
||||
const anim = this.gameObject.getComponent(Animation);
|
||||
anim.play(); // play default clip
|
||||
anim.play("Idle"); // play by clip name
|
||||
anim.stop();
|
||||
anim.loop = true; // loop playback (default: true)
|
||||
anim.playAutomatically = true; // auto-play on enable (default: true)
|
||||
```
|
||||
|
||||
### Animator (state machine — Unity Animator Controller)
|
||||
```ts
|
||||
import { Animator } from "@needle-tools/engine";
|
||||
|
||||
const anim = this.gameObject.getComponent(Animator);
|
||||
|
||||
anim.play("Run"); // play by state name
|
||||
anim.setFloat("Speed", 1.5); // Animator parameters (match Unity parameter names)
|
||||
anim.setBool("IsGrounded", true);
|
||||
anim.setTrigger("Jump");
|
||||
anim.speed = 0.5; // global playback speed multiplier
|
||||
```
|
||||
|
||||
### PlayableDirector (Timeline)
|
||||
```ts
|
||||
import { PlayableDirector } from "@needle-tools/engine";
|
||||
|
||||
const director = this.gameObject.getComponent(PlayableDirector);
|
||||
director.play(); // start playback
|
||||
director.pause();
|
||||
director.stop();
|
||||
director.time = 2.5; // scrub to time (seconds)
|
||||
director.evaluate(); // evaluate at current time (use after setting time)
|
||||
director.isPlaying // check playback state
|
||||
director.isPaused
|
||||
director.duration // total duration in seconds
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Audio
|
||||
|
||||
### AudioSource
|
||||
```ts
|
||||
import { AudioSource } from "@needle-tools/engine";
|
||||
|
||||
const audio = this.gameObject.getComponent(AudioSource);
|
||||
audio.clip = "sounds/music.mp3"; // URL to audio file
|
||||
audio.volume = 0.8;
|
||||
audio.loop = true;
|
||||
audio.spatialBlend = 1; // 0 = 2D, 1 = full 3D positional
|
||||
audio.play();
|
||||
audio.pause();
|
||||
audio.stop();
|
||||
|
||||
// Browser autoplay policy: audio won't play until user interaction
|
||||
AudioSource.registerWaitForAllowAudio(() => {
|
||||
audio.play();
|
||||
});
|
||||
```
|
||||
|
||||
Key properties: `clip` (string/MediaStream), `volume` (0–1), `loop`, `spatialBlend` (0–1), `playOnAwake`, `pitch`, `minDistance`, `maxDistance`, `isPlaying`, `time`, `duration`.
|
||||
|
||||
### AudioListener
|
||||
Represents the "ears" in the scene. Attach to the camera (auto-added to main camera if missing). Only one should be active.
|
||||
|
||||
```ts
|
||||
import { AudioListener } from "@needle-tools/engine";
|
||||
this.context.mainCamera?.addComponent(AudioListener);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Video
|
||||
|
||||
### VideoPlayer
|
||||
```ts
|
||||
import { VideoPlayer } from "@needle-tools/engine";
|
||||
|
||||
const vp = this.gameObject.addComponent(VideoPlayer);
|
||||
vp.url = "videos/intro.mp4"; // mp4, webm, or m3u8 (HLS)
|
||||
vp.isLooping = true;
|
||||
vp.playOnAwake = true;
|
||||
vp.play();
|
||||
vp.pause();
|
||||
vp.stop();
|
||||
vp.currentTime = 10; // seek to 10 seconds
|
||||
|
||||
// Webcam / screen capture:
|
||||
vp.setVideo(mediaStream);
|
||||
|
||||
// HLS livestreams: just set an m3u8 URL — hls.js loads automatically
|
||||
vp.url = "https://stream.example.com/live.m3u8";
|
||||
```
|
||||
|
||||
Key properties: `url`, `isLooping`, `playbackSpeed`, `muted`, `playInBackground`, `screenspace`, `isPlaying`, `videoElement`, `videoTexture`.
|
||||
|
||||
The video texture is applied to the object's material by default (MaterialOverride render mode). The object needs a `Renderer` component.
|
||||
|
||||
---
|
||||
|
||||
## Lighting and Shadows
|
||||
|
||||
### Light
|
||||
```ts
|
||||
import { Light } from "@needle-tools/engine";
|
||||
|
||||
const light = this.gameObject.getComponent(Light);
|
||||
light.intensity = 1.5;
|
||||
light.color.set(1, 0.95, 0.9); // warm white
|
||||
light.shadows = 2; // 0=None, 1=Hard, 2=Soft
|
||||
light.shadowResolution = 2048;
|
||||
```
|
||||
|
||||
Light types (set in Unity/Blender, not changeable at runtime): Directional (1), Point (2), Spot (0). Spot lights have `spotAngle` and `innerSpotAngle`. Point/Spot lights have `range`.
|
||||
|
||||
### ContactShadows
|
||||
Soft ground shadows based on proximity — no lights needed.
|
||||
```ts
|
||||
import { ContactShadows } from "@needle-tools/engine";
|
||||
|
||||
// Auto-create fitted to scene
|
||||
const shadows = ContactShadows.auto(this.context);
|
||||
shadows.opacity = 0.6;
|
||||
shadows.blur = 5;
|
||||
|
||||
// Or via HTML attribute:
|
||||
// <needle-engine contactshadows="0.7">
|
||||
```
|
||||
|
||||
### ShadowCatcher
|
||||
Catches real-time shadows from light sources onto a surface. Use for AR ground planes.
|
||||
```ts
|
||||
import { ShadowCatcher } from "@needle-tools/engine";
|
||||
const catcher = obj.addComponent(ShadowCatcher);
|
||||
catcher.mode = 0; // 0=ShadowMask, 1=Additive, 2=Occluder
|
||||
```
|
||||
|
||||
ContactShadows = soft ambient-style, no lights needed, better performance. ShadowCatcher = accurate shadows from real lights, higher cost.
|
||||
|
||||
### ReflectionProbe
|
||||
Provides per-object environment reflections using cubemap or HDR textures. Objects can reference a specific probe as their reflection source, producing more accurate localized reflections than a single global environment map.
|
||||
|
||||
```ts
|
||||
import { ReflectionProbe } from "@needle-tools/engine";
|
||||
|
||||
// Typically set up in Unity/Blender: add ReflectionProbe to an object, assign a cubemap texture,
|
||||
// then on Renderer components set the probe as "anchor override"
|
||||
|
||||
// Check if a material is using a reflection probe:
|
||||
ReflectionProbe.isUsingReflectionProbe(material);
|
||||
```
|
||||
|
||||
Debug: `?debugreflectionprobe` URL param. Disable all: `?noreflectionprobe`.
|
||||
|
||||
---
|
||||
|
||||
## Post-Processing
|
||||
|
||||
See [postprocessing.md](postprocessing.md) for the full post-processing reference (all effects, parameters, runtime changes).
|
||||
|
||||
Key points: Use `this.context.postprocessing.addEffect(effect)` / `.removeEffect(effect)`. Effects use `VolumeParameter` — set values with `.value`. Toggle with `effect.enabled`. Loads async via `NEEDLE_ENGINE_MODULES.POSTPROCESSING`.
|
||||
|
||||
---
|
||||
|
||||
## Camera
|
||||
|
||||
```ts
|
||||
// Access the main camera
|
||||
this.context.mainCamera // THREE.Camera
|
||||
this.context.mainCameraComponent // Needle Camera component
|
||||
|
||||
// Switch the active camera:
|
||||
import { Camera } from "@needle-tools/engine";
|
||||
const cam = targetObject.getComponent(Camera);
|
||||
this.context.setCurrentCamera(cam); // make this the active camera
|
||||
|
||||
// Camera properties
|
||||
cam.fieldOfView = 60;
|
||||
cam.nearClipPlane = 0.1;
|
||||
cam.farClipPlane = 1000;
|
||||
cam.orthographic = false;
|
||||
|
||||
// Screen to world
|
||||
const ray = cam.screenPointToRay(screenX, screenY);
|
||||
```
|
||||
|
||||
Key properties: `fieldOfView`, `nearClipPlane`, `farClipPlane`, `backgroundColor`, `orthographic`, `orthographicSize`, `clearFlags`, `targetTexture`.
|
||||
|
||||
### Custom camera control (first-person, etc.)
|
||||
For code-only scenes where you want full camera control (first-person, fly cam, etc.):
|
||||
|
||||
1. Use `<needle-engine camera-controls="0">` to prevent auto-added OrbitControls
|
||||
2. Remove any existing OrbitControls — they override camera rotation every frame:
|
||||
```ts
|
||||
import { OrbitControls } from "@needle-tools/engine";
|
||||
|
||||
onStart(ctx => {
|
||||
// Remove OrbitControls so they don't fight your custom camera logic
|
||||
const cam = ctx.mainCamera;
|
||||
const orbit = cam?.getComponent(OrbitControls);
|
||||
if (orbit) orbit.destroy();
|
||||
});
|
||||
```
|
||||
3. Write a `Behaviour` component for camera control — use `update()` and the engine's input system (`this.context.input`), not raw DOM events or `requestAnimationFrame`
|
||||
4. See the [FirstPersonCharacter sample](https://github.com/needle-tools/needle-engine-samples/blob/main/package/Runtime/FirstPersonController/Scripts/FirstPersonController~/FirstPersonCharacter.ts) for a working example
|
||||
|
||||
---
|
||||
|
||||
## Scene Switching
|
||||
|
||||
`SceneSwitcher` manages loading/unloading multiple GLB scenes — useful for multi-room apps, configurators, portfolios.
|
||||
|
||||
```ts
|
||||
import { SceneSwitcher } from "@needle-tools/engine";
|
||||
|
||||
const switcher = this.gameObject.getComponent(SceneSwitcher);
|
||||
await switcher.select(0); // by index
|
||||
await switcher.select("myScene"); // by name/URI
|
||||
await switcher.selectNext();
|
||||
await switcher.selectPrev();
|
||||
|
||||
// Add scenes dynamically
|
||||
switcher.addScene("assets/room2.glb");
|
||||
|
||||
// Events
|
||||
switcher.addEventListener("loadscene-finished", (e) => {
|
||||
console.log("Loaded:", e.detail.scene.url);
|
||||
});
|
||||
```
|
||||
|
||||
Key properties: `scenes` (AssetReference[]), `currentIndex`, `preloadNext`, `preloadPrevious`, `useHistory` (browser back/forward), `useKeyboard` (arrow keys), `useSwipe`, `queryParameterName` (URL param, default `"scene"`).
|
||||
|
||||
You can also implement scene switching yourself using `AssetReference` or `loadAsset()`:
|
||||
```ts
|
||||
import { AssetReference, loadAsset } from "@needle-tools/engine";
|
||||
|
||||
// With AssetReference (caches by URL):
|
||||
const ref = AssetReference.getOrCreate(baseUrl, "assets/room2.glb");
|
||||
const instance = await ref.instantiate({ parent: this.context.scene });
|
||||
|
||||
// With loadAsset (returns a model wrapper):
|
||||
const model = await loadAsset("assets/room2.glb");
|
||||
this.context.scene.add(model.scene);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Interaction
|
||||
|
||||
### DragControls
|
||||
Enables dragging objects in 3D. Automatically takes ownership in networked scenes.
|
||||
```ts
|
||||
import { DragControls, DragMode } from "@needle-tools/engine";
|
||||
const drag = obj.addComponent(DragControls);
|
||||
drag.dragMode = DragMode.XZPlane; // horizontal plane
|
||||
// Modes: XZPlane, Attached, HitNormal, DynamicViewAngle (default), SnapToSurfaces, None
|
||||
```
|
||||
|
||||
### Duplicatable
|
||||
Add alongside `DragControls` — dragging creates a clone instead of moving the original.
|
||||
```ts
|
||||
import { Duplicatable } from "@needle-tools/engine";
|
||||
obj.addComponent(Duplicatable);
|
||||
```
|
||||
|
||||
### DropListener
|
||||
Enables drag-and-drop of files from the desktop into the 3D scene (GLB, FBX, OBJ, USDZ, VRM, images).
|
||||
```ts
|
||||
import { DropListener } from "@needle-tools/engine";
|
||||
const dl = myObject.addComponent(DropListener);
|
||||
dl.fitIntoVolume = true; // auto-scale dropped objects
|
||||
dl.useNetworking = true; // sync drops to other clients
|
||||
|
||||
// Or load programmatically:
|
||||
const loaded = await dl.loadFromURL("https://example.com/model.glb");
|
||||
```
|
||||
|
||||
### CharacterController
|
||||
Capsule collider + rigidbody for character movement. Auto-creates physics components on enable.
|
||||
```ts
|
||||
import { CharacterController } from "@needle-tools/engine";
|
||||
|
||||
const cc = this.gameObject.getComponent(CharacterController);
|
||||
cc.move(new Vector3(0, 0, 0.1)); // move forward
|
||||
cc.isGrounded; // true when touching ground
|
||||
|
||||
// For jumping, use the rigidbody directly:
|
||||
if (cc.isGrounded) cc.rigidbody.applyImpulse(new Vector3(0, 5, 0));
|
||||
```
|
||||
|
||||
`CharacterControllerInput` provides a ready-made WASD + Space control scheme with double-jump and animator integration.
|
||||
|
||||
For a full first-person controller example, see the [FirstPersonCharacter sample](https://github.com/needle-tools/needle-engine-samples/blob/main/package/Runtime/FirstPersonController/Scripts/FirstPersonController~/FirstPersonCharacter.ts).
|
||||
|
||||
For clickable hotspot labels on 3D objects (common in product configurators), see the [Hotspot sample](https://github.com/needle-tools/needle-engine-samples/blob/main/package/Runtime/Hotspots/Scripts/Needle.Hotspots~/Hotspot.ts).
|
||||
|
||||
### needle-menu (built-in UI menu)
|
||||
The `<needle-menu>` web component provides a built-in hamburger menu. Components like `SyncedRoom` and `Voip` add buttons to it automatically. Access via `this.context.menu`.
|
||||
|
||||
```ts
|
||||
// Add a button using ButtonInfo object (recommended)
|
||||
this.context.menu.appendChild({
|
||||
label: "My Action",
|
||||
icon: "settings", // Google Material Icons name
|
||||
onClick: () => { /* ... */ },
|
||||
priority: 50, // higher = further right, always visible
|
||||
});
|
||||
|
||||
// Or add a raw HTML button
|
||||
const button = document.createElement("button");
|
||||
button.textContent = "Click me";
|
||||
button.onclick = () => { /* ... */ };
|
||||
this.context.menu.appendChild(button);
|
||||
|
||||
// Control visibility (hiding requires Needle Engine PRO license in production)
|
||||
this.context.menu.setVisible(false);
|
||||
|
||||
// Hide the Needle logo (requires license)
|
||||
this.context.menu.showNeedleLogo(false);
|
||||
|
||||
// Set button priority (controls ordering and which buttons stay visible when space is limited)
|
||||
NeedleMenu.setElementPriority(button, 90);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Splines
|
||||
|
||||
### SplineContainer
|
||||
Defines curves/paths in the scene. Can be created in Unity/Blender or from code.
|
||||
```ts
|
||||
import { SplineContainer } from "@needle-tools/engine";
|
||||
import { Vector3 } from "three";
|
||||
|
||||
const spline = obj.addComponent(SplineContainer);
|
||||
spline.addKnot({ position: new Vector3(0, 0, 0) })
|
||||
.addKnot({ position: new Vector3(5, 2, 5) })
|
||||
.addKnot({ position: new Vector3(10, 0, 0) });
|
||||
spline.closed = false;
|
||||
|
||||
// Sample the spline (t: 0–1)
|
||||
const point = spline.getPointAt(0.5); // world-space position
|
||||
const tangent = spline.getTangentAt(0.5); // world-space tangent
|
||||
```
|
||||
|
||||
### SplineWalker
|
||||
Moves an object along a spline path.
|
||||
```ts
|
||||
import { SplineWalker } from "@needle-tools/engine";
|
||||
const walker = obj.addComponent(SplineWalker);
|
||||
walker.spline = splineContainer;
|
||||
walker.duration = 5; // seconds for full traversal
|
||||
walker.autoRun = true;
|
||||
walker.useLookAt = true; // face movement direction
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Debug Tools
|
||||
|
||||
### Gizmos
|
||||
Static methods for runtime debug drawing — shapes auto-remove after a duration (0 = one frame).
|
||||
```ts
|
||||
import { Gizmos } from "@needle-tools/engine";
|
||||
|
||||
Gizmos.DrawLine(start, end, color, duration, depthTest);
|
||||
Gizmos.DrawWireSphere(center, radius, color, duration);
|
||||
Gizmos.DrawRay(origin, direction, color, duration);
|
||||
Gizmos.DrawLabel(position, text, size, duration);
|
||||
Gizmos.DrawArrow(start, end, color, duration);
|
||||
Gizmos.DrawWireBox(center, size, color, duration);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Utilities
|
||||
|
||||
### EventList (Unity Events)
|
||||
`EventList` is how Unity Events are serialized and invoked at runtime. Declare with `@serializable(EventList)` and call `.invoke()`.
|
||||
```ts
|
||||
import { EventList, serializable } from "@needle-tools/engine";
|
||||
|
||||
@serializable(EventList) onClick?: EventList;
|
||||
|
||||
// Invoke from code:
|
||||
this.onClick?.invoke();
|
||||
|
||||
// Subscribe from code:
|
||||
const unsub = this.onClick?.addEventListener(() => console.log("Clicked!"));
|
||||
unsub(); // unsubscribe
|
||||
```
|
||||
|
||||
### Creating Objects from Code
|
||||
`ObjectUtils` provides convenience methods for creating primitives and text. These are helpers — you can always use standard Three.js objects directly (`new Mesh(geometry, material)`).
|
||||
```ts
|
||||
import { ObjectUtils, PrimitiveType } from "@needle-tools/engine";
|
||||
|
||||
const cube = ObjectUtils.createPrimitive(PrimitiveType.Cube, {
|
||||
color: 0xff0000,
|
||||
parent: this.gameObject,
|
||||
position: { x: 0, y: 1, z: 0 }
|
||||
});
|
||||
|
||||
const text = ObjectUtils.createText("Hello World");
|
||||
this.context.scene.add(text);
|
||||
```
|
||||
|
||||
Available primitives: `Cube`, `Sphere`, `Quad`, `Cylinder`. For anything more complex, use Three.js geometry directly or load GLB models.
|
||||
|
||||
### ParticleSystem
|
||||
Full particle system with emission, shape, velocity, color/size over lifetime modules. Currently best configured via Unity/Blender — difficult to set up from code only.
|
||||
```ts
|
||||
import { ParticleSystem } from "@needle-tools/engine";
|
||||
const ps = this.gameObject.getComponent(ParticleSystem);
|
||||
ps.play();
|
||||
ps.stop();
|
||||
ps.pause();
|
||||
```
|
||||
47
.agents/skills/needle-engine/references/deployment.md
Normal file
47
.agents/skills/needle-engine/references/deployment.md
Normal file
@@ -0,0 +1,47 @@
|
||||
# Needle Engine — Deployment Reference
|
||||
|
||||
## Needle Cloud (recommended)
|
||||
|
||||
### GitHub Actions (deploy-on-push)
|
||||
Use the official GitHub Action — do NOT use `npx needle-cloud deploy` in CI (there is no `--non-interactive` flag):
|
||||
|
||||
```yaml
|
||||
# .github/workflows/deploy.yml
|
||||
name: Deploy to Needle Cloud
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with: { node-version: 22 }
|
||||
- run: npm ci
|
||||
- run: npm run build
|
||||
- uses: needle-tools/deploy-to-needle-cloud-action@v1
|
||||
with:
|
||||
token: ${{ secrets.NEEDLE_CLOUD_TOKEN }}
|
||||
dir: ./dist
|
||||
# name: my-project # optional — defaults to the repo name
|
||||
# webhookUrl: ${{ secrets.DISCORD_WEBHOOK_URL }} # optional — Discord/Slack deploy notifications
|
||||
```
|
||||
|
||||
Create a `NEEDLE_CLOUD_TOKEN` secret in your repo settings (get the token from https://cloud.needle.tools/team with read/write permissions).
|
||||
|
||||
### CLI deployment (manual)
|
||||
```bash
|
||||
# Auth: run `npx needle-cloud login`, or set NEEDLE_CLOUD_TOKEN env var
|
||||
# For CI/CD: create an access token at https://cloud.needle.tools/team (read/write permissions)
|
||||
npx needle-cloud deploy dist --name my-project # ALWAYS pass --name (defaults to "index" otherwise)
|
||||
npx needle-cloud deploy dist # ⚠️ avoid: project will be named "index"
|
||||
npx needle-cloud deploy dist --team my-team-name # deploy to a specific team
|
||||
npx needle-cloud deploy dist --token # prompts to paste an access token
|
||||
```
|
||||
|
||||
## Other platforms
|
||||
|
||||
Vercel, Netlify, GitHub Pages, itch.io, FTP — all work as standard static site deployments. Networking works on any platform — Needle provides the networking server by default. Self-hosting the networking server is available on request for PRO/Enterprise users.
|
||||
|
||||
See the [deployment docs](https://engine.needle.tools/docs/how-to-guides/deployment/) for platform-specific guides.
|
||||
276
.agents/skills/needle-engine/references/examples.md
Normal file
276
.agents/skills/needle-engine/references/examples.md
Normal file
@@ -0,0 +1,276 @@
|
||||
# Needle Engine — Component Examples
|
||||
|
||||
Practical examples of common component patterns. All components extend `Behaviour` from `@needle-tools/engine`.
|
||||
|
||||
---
|
||||
|
||||
## Rotate an object
|
||||
|
||||
```ts
|
||||
import { Behaviour, serializable, registerType } from "@needle-tools/engine";
|
||||
|
||||
@registerType
|
||||
export class Rotate extends Behaviour {
|
||||
@serializable() speed: number = 1;
|
||||
|
||||
update() {
|
||||
this.gameObject.rotation.y += this.speed * this.context.time.deltaTime;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Respond to clicks
|
||||
|
||||
```ts
|
||||
import { Behaviour, registerType } from "@needle-tools/engine";
|
||||
import type { PointerEventData } from "@needle-tools/engine";
|
||||
|
||||
@registerType
|
||||
export class ClickHandler extends Behaviour {
|
||||
|
||||
onPointerClick(args: PointerEventData) {
|
||||
console.log("Clicked:", this.gameObject.name);
|
||||
// Scale up briefly on click
|
||||
this.gameObject.scale.multiplyScalar(1.2);
|
||||
setTimeout(() => this.gameObject.scale.multiplyScalar(1 / 1.2), 200);
|
||||
}
|
||||
|
||||
// Other pointer events: onPointerEnter, onPointerExit, onPointerDown, onPointerUp, onPointerMove
|
||||
}
|
||||
```
|
||||
|
||||
Pointer events require an `EventSystem` and a `Raycaster` component in the scene (both are included by default in Unity/Blender exports).
|
||||
|
||||
---
|
||||
|
||||
## Load a GLB at runtime
|
||||
|
||||
```ts
|
||||
import { Behaviour, AssetReference, serializable, registerType } from "@needle-tools/engine";
|
||||
import { Object3D } from "three";
|
||||
|
||||
@registerType
|
||||
export class RuntimeLoader extends Behaviour {
|
||||
// Set in Unity/Blender inspector, or assign from code
|
||||
@serializable(AssetReference) model?: AssetReference;
|
||||
|
||||
async start() {
|
||||
// Option 1: From a serialized AssetReference
|
||||
if (this.model) {
|
||||
const instance = await this.model.instantiate({ parent: this.gameObject });
|
||||
}
|
||||
|
||||
// Option 2: From a URL (code-only)
|
||||
const ref = AssetReference.getOrCreate(this.context.domElement.baseURI, "assets/chair.glb");
|
||||
const chair = await ref.instantiate({ parent: this.gameObject });
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Synced multiplayer state
|
||||
|
||||
```ts
|
||||
import { Behaviour, serializable, registerType, syncField } from "@needle-tools/engine";
|
||||
|
||||
@registerType
|
||||
export class SyncedCounter extends Behaviour {
|
||||
// @syncField handles networking sync; add @serializable() too if the field should also
|
||||
// deserialize from GLB (when set in Unity/Blender). For code-only components, @syncField alone is fine.
|
||||
@syncField(SyncedCounter.prototype.onCountChanged)
|
||||
count: number = 0;
|
||||
|
||||
private onCountChanged(newValue: number, oldValue: number) {
|
||||
console.log(`Count changed: ${oldValue} → ${newValue}`);
|
||||
}
|
||||
|
||||
increment() {
|
||||
this.count += 1; // reassignment triggers sync
|
||||
}
|
||||
|
||||
// For arrays/objects: must reassign to trigger sync
|
||||
@syncField() tags: string[] = [];
|
||||
|
||||
addTag(tag: string) {
|
||||
this.tags.push(tag);
|
||||
this.tags = this.tags; // force sync
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Change materials at runtime
|
||||
|
||||
```ts
|
||||
import { Behaviour, Renderer, registerType } from "@needle-tools/engine";
|
||||
import { MeshStandardMaterial, Color } from "three";
|
||||
|
||||
@registerType
|
||||
export class ColorChanger extends Behaviour {
|
||||
|
||||
onPointerClick() {
|
||||
// Option 1: Via Renderer component (if available, e.g. from Unity/Blender export)
|
||||
const renderer = this.gameObject.getComponent(Renderer);
|
||||
if (renderer?.sharedMaterial) {
|
||||
(renderer.sharedMaterial as MeshStandardMaterial).color = new Color(Math.random(), Math.random(), Math.random());
|
||||
}
|
||||
|
||||
// Option 2: Direct Three.js access (always works)
|
||||
this.gameObject.traverse(child => {
|
||||
if ((child as any).material) {
|
||||
(child as any).material.color = new Color(Math.random(), Math.random(), Math.random());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Set up a scene from code (no Unity/Blender)
|
||||
|
||||
Use `onStart` to safely access the context — never poll with `setInterval`.
|
||||
|
||||
```ts
|
||||
import { onStart, ObjectUtils, PrimitiveType, ContactShadows, SyncedRoom } from "@needle-tools/engine";
|
||||
import { DirectionalLight, AmbientLight } from "three";
|
||||
|
||||
onStart(ctx => {
|
||||
// Lighting
|
||||
const dirLight = new DirectionalLight(0xffffff, 2);
|
||||
dirLight.position.set(5, 10, 5);
|
||||
dirLight.castShadow = true;
|
||||
ctx.scene.add(dirLight);
|
||||
ctx.scene.add(new AmbientLight(0xffffff, 0.5));
|
||||
|
||||
// Ground
|
||||
const ground = ObjectUtils.createPrimitive(PrimitiveType.Cube, {
|
||||
color: 0x888888,
|
||||
scale: { x: 10, y: 0.1, z: 10 }
|
||||
});
|
||||
ctx.scene.add(ground);
|
||||
|
||||
// Some objects
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const sphere = ObjectUtils.createPrimitive(PrimitiveType.Sphere, {
|
||||
color: Math.random() * 0xffffff,
|
||||
position: { x: (i - 2) * 2, y: 1, z: 0 }
|
||||
});
|
||||
ctx.scene.add(sphere);
|
||||
}
|
||||
|
||||
// Shadows
|
||||
ContactShadows.auto(ctx);
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Keyboard input
|
||||
|
||||
```ts
|
||||
import { Behaviour, registerType } from "@needle-tools/engine";
|
||||
import { Vector3 } from "three";
|
||||
|
||||
@registerType
|
||||
export class KeyboardMover extends Behaviour {
|
||||
speed: number = 5;
|
||||
|
||||
update() {
|
||||
const dt = this.context.time.deltaTime;
|
||||
const input = this.context.input;
|
||||
|
||||
// Key codes: use KeyCode values ("KeyW", "Space", "ArrowLeft") or lowercase letters ("w")
|
||||
if (input.getKeyPressed("KeyW")) this.gameObject.position.z -= this.speed * dt;
|
||||
if (input.getKeyPressed("KeyS")) this.gameObject.position.z += this.speed * dt;
|
||||
if (input.getKeyPressed("KeyA")) this.gameObject.position.x -= this.speed * dt;
|
||||
if (input.getKeyPressed("KeyD")) this.gameObject.position.x += this.speed * dt;
|
||||
|
||||
if (input.getKeyDown("Space")) {
|
||||
console.log("Jump!");
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## First-person camera movement
|
||||
|
||||
Three.js cameras look down `-Z`, so `getWorldDirection` returns the negated forward. Use `worldForward` from Needle's Object3D extensions instead — it handles this correctly. Note: `worldForward` works on `ctx.mainCamera` because Needle patches all Object3D instances, including Three.js cameras.
|
||||
|
||||
```ts
|
||||
import { Behaviour, registerType } from "@needle-tools/engine";
|
||||
import { Vector3 } from "three";
|
||||
|
||||
@registerType
|
||||
export class FirstPersonMove extends Behaviour {
|
||||
speed: number = 5;
|
||||
private _forward = new Vector3();
|
||||
private _right = new Vector3();
|
||||
|
||||
update() {
|
||||
const dt = this.context.time.deltaTime;
|
||||
const input = this.context.input;
|
||||
const cam = this.context.mainCamera;
|
||||
if (!cam) return;
|
||||
|
||||
// Use Needle's worldForward/worldRight — correctly handles Three.js -Z convention
|
||||
this._forward.copy(cam.worldForward);
|
||||
this._forward.y = 0;
|
||||
this._forward.normalize();
|
||||
|
||||
this._right.copy(cam.worldRight);
|
||||
this._right.y = 0;
|
||||
this._right.normalize();
|
||||
|
||||
let moveX = 0, moveZ = 0;
|
||||
|
||||
if (input.getKeyPressed("KeyW")) { moveX += this._forward.x; moveZ += this._forward.z; }
|
||||
if (input.getKeyPressed("KeyS")) { moveX -= this._forward.x; moveZ -= this._forward.z; }
|
||||
if (input.getKeyPressed("KeyA")) { moveX -= this._right.x; moveZ -= this._right.z; }
|
||||
if (input.getKeyPressed("KeyD")) { moveX += this._right.x; moveZ += this._right.z; }
|
||||
|
||||
const len = Math.sqrt(moveX * moveX + moveZ * moveZ);
|
||||
if (len > 0) {
|
||||
cam.position.x += (moveX / len) * this.speed * dt;
|
||||
cam.position.z += (moveZ / len) * this.speed * dt;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Coroutine (timed sequence)
|
||||
|
||||
```ts
|
||||
import { Behaviour, registerType, WaitForSeconds } from "@needle-tools/engine";
|
||||
|
||||
@registerType
|
||||
export class TrafficLight extends Behaviour {
|
||||
|
||||
start() {
|
||||
this.startCoroutine(this.cycle());
|
||||
}
|
||||
|
||||
*cycle() {
|
||||
while (true) {
|
||||
this.setColor("green");
|
||||
yield WaitForSeconds(5);
|
||||
this.setColor("yellow");
|
||||
yield WaitForSeconds(2);
|
||||
this.setColor("red");
|
||||
yield WaitForSeconds(5);
|
||||
}
|
||||
}
|
||||
|
||||
private setColor(color: string) {
|
||||
console.log("Light:", color);
|
||||
}
|
||||
}
|
||||
```
|
||||
166
.agents/skills/needle-engine/references/integration.md
Normal file
166
.agents/skills/needle-engine/references/integration.md
Normal file
@@ -0,0 +1,166 @@
|
||||
# Needle Engine — Framework Integration
|
||||
|
||||
## React
|
||||
|
||||
### Listen for 3D events in a component
|
||||
```tsx
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
function ScoreDisplay() {
|
||||
const [score, setScore] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
const ne = document.querySelector("needle-engine");
|
||||
const handler = (e: CustomEvent) => setScore(e.detail.score);
|
||||
ne?.addEventListener("score-changed", handler as EventListener);
|
||||
return () => ne?.removeEventListener("score-changed", handler as EventListener);
|
||||
}, []);
|
||||
|
||||
return <div>Score: {score}</div>;
|
||||
}
|
||||
```
|
||||
|
||||
### Call into the 3D scene from React
|
||||
```tsx
|
||||
function GameControls() {
|
||||
const addScore = async () => {
|
||||
const { findObjectOfType } = await import("@needle-tools/engine");
|
||||
const { MyScoreManager } = await import("./scripts/MyScoreManager.js");
|
||||
findObjectOfType(MyScoreManager)?.addScore(10);
|
||||
};
|
||||
return <button onClick={addScore}>+10</button>;
|
||||
}
|
||||
```
|
||||
|
||||
### Use engine hooks from React
|
||||
```tsx
|
||||
useEffect(() => {
|
||||
import("@needle-tools/engine").then(({ onStart }) => {
|
||||
onStart((ctx) => {
|
||||
// safe to access components here
|
||||
});
|
||||
});
|
||||
}, []);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Svelte / SvelteKit
|
||||
|
||||
```svelte
|
||||
<script>
|
||||
import { onMount } from "svelte";
|
||||
let score = 0;
|
||||
|
||||
onMount(async () => {
|
||||
// Dynamic import required for SSR — engine needs browser APIs
|
||||
const { onStart } = await import("@needle-tools/engine");
|
||||
|
||||
// onStart fires once when the context/scene is ready — never poll with setInterval
|
||||
onStart(ctx => {
|
||||
// Safe to access components, add components, etc.
|
||||
console.log("Scene ready:", ctx.scene);
|
||||
});
|
||||
|
||||
// Listen for custom events from 3D components
|
||||
const ne = document.querySelector("needle-engine");
|
||||
const handler = (e) => (score = e.detail.score);
|
||||
ne?.addEventListener("score-changed", handler);
|
||||
return () => ne?.removeEventListener("score-changed", handler);
|
||||
});
|
||||
|
||||
async function addScore() {
|
||||
const { findObjectOfType, Context } = await import("@needle-tools/engine");
|
||||
const { MyScoreManager } = await import("../scripts/MyScoreManager.js");
|
||||
findObjectOfType(MyScoreManager, Context.Current)?.addScore(10);
|
||||
}
|
||||
</script>
|
||||
|
||||
<p>Score: {score}</p>
|
||||
<button on:click={addScore}>+10</button>
|
||||
<needle-engine src="assets/scene.glb" />
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Vue / Nuxt
|
||||
|
||||
```vue
|
||||
<template>
|
||||
<div>Score: {{ score }}</div>
|
||||
<needle-engine src="assets/scene.glb" ref="ne" />
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, onMounted, onUnmounted } from "vue";
|
||||
|
||||
const score = ref(0);
|
||||
const ne = ref(null);
|
||||
|
||||
function onScore(e) { score.value = e.detail.score; }
|
||||
|
||||
onMounted(() => {
|
||||
ne.value?.addEventListener("score-changed", onScore);
|
||||
import("@needle-tools/engine").then(({ onStart }) => {
|
||||
onStart((ctx) => {
|
||||
// safe to access components here
|
||||
});
|
||||
});
|
||||
});
|
||||
onUnmounted(() => ne.value?.removeEventListener("score-changed", onScore));
|
||||
</script>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Vanilla JS / No Framework
|
||||
|
||||
```html
|
||||
<needle-engine src="assets/scene.glb"></needle-engine>
|
||||
|
||||
<script type="module">
|
||||
import { onStart, onUpdate } from "@needle-tools/engine";
|
||||
|
||||
onStart((ctx) => {
|
||||
console.log("Scene ready:", ctx.scene);
|
||||
});
|
||||
</script>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Engine Hooks Reference
|
||||
|
||||
These standalone functions from `@needle-tools/engine` mirror the component lifecycle but work outside of a class:
|
||||
|
||||
| Hook | When it fires |
|
||||
|---|---|
|
||||
| `onInitialized(cb)` | Once after context creation and first content load |
|
||||
| `onStart(cb)` | Once when the context/scene is ready |
|
||||
| `onUpdate(cb)` | Every frame (before rendering) |
|
||||
| `onBeforeRender(cb)` | Just before Three.js renders |
|
||||
| `onAfterRender(cb)` | Just after Three.js renders |
|
||||
| `onClear(cb)` | Before context is cleared (e.g. when `src` changes) |
|
||||
| `onDestroy(cb)` | When the context is torn down |
|
||||
|
||||
All callbacks receive `(ctx: Context)` as their argument.
|
||||
|
||||
### Client-only (no SSR)
|
||||
When server-side rendering is **disabled**, import and call hooks directly:
|
||||
```ts
|
||||
import { onStart, onUpdate, onBeforeRender, onDestroy } from "@needle-tools/engine";
|
||||
|
||||
onStart((ctx) => { /* setup */ });
|
||||
onUpdate((ctx) => { /* per-frame logic */ });
|
||||
onDestroy((ctx) => { /* cleanup */ });
|
||||
```
|
||||
|
||||
### With SSR (Next.js, SvelteKit, Nuxt, etc.)
|
||||
`@needle-tools/engine` depends on WebGL / browser APIs and **cannot be imported on the server**. Use a dynamic import so the module is only loaded client-side (same pattern as with any three.js-based engine):
|
||||
```ts
|
||||
import("@needle-tools/engine").then(({ onStart, onUpdate, onDestroy }) => {
|
||||
onStart((ctx) => { /* setup */ });
|
||||
onUpdate((ctx) => { /* per-frame logic */ });
|
||||
onDestroy((ctx) => { /* cleanup */ });
|
||||
});
|
||||
```
|
||||
411
.agents/skills/needle-engine/references/networking.md
Normal file
411
.agents/skills/needle-engine/references/networking.md
Normal file
@@ -0,0 +1,411 @@
|
||||
# Needle Engine — Networking Reference
|
||||
|
||||
## Table of Contents
|
||||
- [Core: context.connection](#core-thiscontextconnection)
|
||||
- [Persistent vs ephemeral messages](#persistent-vs-ephemeral-messages-guid)
|
||||
- [SyncedRoom](#syncedroom-convenience-component)
|
||||
- [@syncField](#syncfield-auto-sync-fields)
|
||||
- [SyncedTransform](#syncedtransform-sync-positionrotation)
|
||||
- [PlayerSync + PlayerState](#playersync--playerstate-player-avatar-management)
|
||||
- [Voip and ScreenCapture](#voice--video-voip-and-screencapture)
|
||||
|
||||
---
|
||||
|
||||
> Some APIs documented here (e.g. Voip volume/speaking detection, PlayerSync.setupFrom with Object3D) may require the latest pre-release version of `@needle-tools/engine`.
|
||||
|
||||
Needle Engine networking is layered. The lowest level is `this.context.connection` (WebSocket rooms + messages). Higher-level components build on it.
|
||||
|
||||
## Core: `this.context.connection`
|
||||
```ts
|
||||
// Connect and join a room — no SyncedRoom needed
|
||||
this.context.connection.connect();
|
||||
this.context.connection.joinRoom("my-room");
|
||||
|
||||
// Room state
|
||||
this.context.connection.isConnected // boolean
|
||||
this.context.connection.isInRoom // boolean
|
||||
this.context.connection.connectionId // this client's ID
|
||||
this.context.connection.usersInRoom() // all user IDs in current room
|
||||
|
||||
// Send and receive custom messages
|
||||
this.context.connection.send("my-event", { score: 10, name: "Alice" });
|
||||
this.context.connection.beginListen("my-event", (msg: { score: number; name: string }) => {
|
||||
console.log(msg.score, msg.name);
|
||||
});
|
||||
this.context.connection.stopListen("my-event", handler); // always clean up in onDisable/onDestroy
|
||||
|
||||
// Room lifecycle events (import { RoomEvents } from "@needle-tools/engine")
|
||||
this.context.connection.beginListen(RoomEvents.JoinedRoom, () => { ... });
|
||||
this.context.connection.beginListen(RoomEvents.LeftRoom, () => { ... });
|
||||
this.context.connection.beginListen(RoomEvents.UserJoinedRoom, (evt) => {
|
||||
console.log("User joined:", evt.userId); // evt: { userId: string }
|
||||
});
|
||||
this.context.connection.beginListen(RoomEvents.UserLeftRoom, (evt) => {
|
||||
console.log("User left:", evt.userId); // evt: { userId: string }
|
||||
});
|
||||
this.context.connection.beginListen(RoomEvents.RoomStateSent, () => { ... }); // all persisted state received
|
||||
|
||||
// Always check connection state before sending:
|
||||
if (this.context.connection.isInRoom) {
|
||||
this.context.connection.send("my-event", { data: 42 });
|
||||
}
|
||||
```
|
||||
|
||||
**Important:** `send()` broadcasts to all users in the room **except yourself** — you won't receive your own messages. Custom messages do NOT automatically include a sender ID. If you need to identify who sent a message (e.g. to map data to a specific player), include `connectionId` yourself:
|
||||
```ts
|
||||
this.context.connection.send("player-score", {
|
||||
senderId: this.context.connection.connectionId,
|
||||
score: 42,
|
||||
});
|
||||
|
||||
this.context.connection.beginListen("player-score", (msg) => {
|
||||
// msg.senderId tells you which player sent this
|
||||
playerScores.set(msg.senderId, msg.score);
|
||||
});
|
||||
```
|
||||
|
||||
`userId` is only available in room lifecycle events (`UserJoinedRoom`, `UserLeftRoom`), not in custom messages.
|
||||
|
||||
---
|
||||
|
||||
## Networked Instantiation and Destruction
|
||||
|
||||
For spawning objects that should appear on all clients, use `syncInstantiate` / `instantiateSynced` instead of manually sending custom events.
|
||||
|
||||
```ts
|
||||
import { instantiate, syncInstantiate, syncDestroy, registerPrefabProvider } from "@needle-tools/engine";
|
||||
|
||||
// Local clone (only on this client)
|
||||
const clone = instantiate(prefabObject, { parent: this.gameObject });
|
||||
|
||||
// Networked spawn (appears on all connected clients)
|
||||
const networked = syncInstantiate(prefabObject, {
|
||||
parent: this.gameObject,
|
||||
position: [x, y, z],
|
||||
deleteOnDisconnect: true, // removed when the spawning user disconnects
|
||||
});
|
||||
|
||||
// Persistent networked spawn (survives disconnects, replayed to late joiners)
|
||||
const persistent = syncInstantiate(prefabObject, {
|
||||
parent: this.gameObject,
|
||||
deleteOnDisconnect: false,
|
||||
});
|
||||
|
||||
// Via AssetReference
|
||||
const synced = await myAssetRef.instantiateSynced({
|
||||
parent: this.gameObject,
|
||||
deleteOnDisconnect: false,
|
||||
});
|
||||
|
||||
// Networked destroy (removed on all clients)
|
||||
syncDestroy(obj, this.context.connection, true);
|
||||
|
||||
// Listen for remote syncInstantiate events (get references to objects spawned by other users)
|
||||
import { onSyncInstantiate } from "@needle-tools/engine";
|
||||
const unsub = onSyncInstantiate((instance, model) => {
|
||||
console.log("Remote object created:", instance.name, model.originalGuid);
|
||||
});
|
||||
// later: unsub();
|
||||
```
|
||||
|
||||
### Runtime prefabs with `registerPrefabProvider`
|
||||
For runtime-created prefabs (not loaded from GLB), **every client** must register the prefab in their setup code — not just the client that calls `syncInstantiate`. This is because late joiners receive state replay and need to resolve the prefab by guid locally.
|
||||
|
||||
```ts
|
||||
import { registerPrefabProvider, ObjectUtils, syncInstantiate } from "@needle-tools/engine";
|
||||
|
||||
// ALL clients run this setup code:
|
||||
const cookiePrefab = ObjectUtils.createPrimitive("Cube", { color: 0xff8c00 });
|
||||
cookiePrefab.guid = "cookie-prefab";
|
||||
registerPrefabProvider("cookie-prefab", async () => cookiePrefab);
|
||||
|
||||
// Only the first player calls syncInstantiate — late joiners get state replay
|
||||
syncInstantiate(cookiePrefab, { parent: ctx.scene, deleteOnDisconnect: false });
|
||||
```
|
||||
|
||||
Note: `syncInstantiate` auto-registers the prefab on the calling client, but **remote clients and late joiners** still need the explicit `registerPrefabProvider` call in their setup code.
|
||||
|
||||
### PlayerSync with runtime-created avatars (no GLB)
|
||||
Since Needle Engine 5.0.1, `PlayerSync.setupFrom` accepts an Object3D directly — no GLB URL needed:
|
||||
```ts
|
||||
import { PlayerSync, PlayerState, ObjectUtils } from "@needle-tools/engine";
|
||||
|
||||
// Create your avatar template
|
||||
const avatarPrefab = ObjectUtils.createPrimitive("Sphere", { color: 0x4488ff });
|
||||
|
||||
// Pass the Object3D directly — PlayerState is added automatically
|
||||
const ps = await PlayerSync.setupFrom(avatarPrefab, { guid: "player-avatar" });
|
||||
ctx.scene.add(ps.gameObject);
|
||||
|
||||
// onPlayerSpawned fires for both local and remote players
|
||||
ps.onPlayerSpawned?.addEventListener((playerObj) => {
|
||||
// playerObj is the spawned avatar Object3D
|
||||
// Use PlayerState.isLocalPlayer(playerObj) to check if it's yours
|
||||
});
|
||||
```
|
||||
|
||||
For older versions, use `registerPrefabProvider` with a URL key:
|
||||
```ts
|
||||
import { PlayerSync, PlayerState, registerPrefabProvider, ObjectUtils, GameObject } from "@needle-tools/engine";
|
||||
|
||||
const avatarPrefab = ObjectUtils.createPrimitive("Sphere", { color: 0x4488ff });
|
||||
GameObject.addComponent(avatarPrefab, PlayerState);
|
||||
|
||||
const avatarKey = "runtime://player-avatar";
|
||||
registerPrefabProvider(avatarKey, async () => avatarPrefab);
|
||||
const ps = await PlayerSync.setupFrom(avatarKey);
|
||||
ctx.scene.add(ps.gameObject);
|
||||
```
|
||||
|
||||
### World-building pattern (first player seeds, late joiners receive)
|
||||
```ts
|
||||
let shouldBuildWorld = false;
|
||||
|
||||
connection.beginListen(RoomEvents.JoinedRoom, () => {
|
||||
const inRoom = connection.usersInRoom();
|
||||
shouldBuildWorld = inRoom.length === 1; // I'm the only one here
|
||||
});
|
||||
|
||||
connection.beginListen(RoomEvents.RoomStateSent, () => {
|
||||
// State replay complete — only build if we're first AND no objects exist yet
|
||||
if (!shouldBuildWorld) return;
|
||||
|
||||
for (let i = 0; i < 100; i++) {
|
||||
syncInstantiate(cookiePrefab, {
|
||||
parent: ctx.scene,
|
||||
position: [x, 0, z],
|
||||
deleteOnDisconnect: false, // persists for late joiners
|
||||
});
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Persistent vs ephemeral messages (guid)
|
||||
When a message's `data` contains a `guid` field, the server stores it as room state. New users joining later receive all stored state via `RoomStateSent`. Messages without a `guid` are fire-and-forget — only currently connected users see them.
|
||||
|
||||
```ts
|
||||
// Ephemeral — only users currently in the room receive this
|
||||
this.context.connection.send("chat", { text: "hello", sender: "Alice" });
|
||||
|
||||
// Persistent — server stores this by guid; late joiners get it automatically
|
||||
this.context.connection.send("object-color", { guid: this.guid, color: "#ff0000" });
|
||||
|
||||
// Read cached state for a guid (received from server or local sends)
|
||||
const state = this.context.connection.tryGetState(this.guid);
|
||||
|
||||
// Delete persisted state (removes from server so new joiners won't get it)
|
||||
this.context.connection.sendDeleteRemoteState(this.guid);
|
||||
|
||||
// Delete ALL room state (use with caution)
|
||||
this.context.connection.sendDeleteRemoteStateAll();
|
||||
```
|
||||
|
||||
Any JSON message with a `guid` can also include these optional fields:
|
||||
```ts
|
||||
this.context.connection.send("my-state", {
|
||||
guid: this.guid, // persists on server
|
||||
dontSave: false, // set true to prevent server storage (ephemeral but with guid for identity)
|
||||
deleteOnDisconnect: true, // auto-delete when sender disconnects
|
||||
// ...your data
|
||||
});
|
||||
```
|
||||
|
||||
This is how `@syncField()` and `SyncedTransform` work under the hood — they send messages with the component's `guid`, so state persists for late joiners. Understanding this lets you build custom networking that also persists correctly.
|
||||
|
||||
---
|
||||
|
||||
## SyncedRoom (convenience component)
|
||||
Wraps `context.connection` with auto-join, URL params, random rooms, auto-reconnect, and a join/leave menu button. Add to any object — no code needed for basic room management.
|
||||
```ts
|
||||
import { SyncedRoom } from "@needle-tools/engine";
|
||||
|
||||
// Add at runtime:
|
||||
myObject.addComponent(SyncedRoom, { roomName: "my-room" });
|
||||
// or join a random room:
|
||||
myObject.addComponent(SyncedRoom, { joinRandomRoom: true });
|
||||
// or with a prefix (useful for multiple apps on same server):
|
||||
myObject.addComponent(SyncedRoom, { joinRandomRoom: true, roomPrefix: "myApp_" });
|
||||
```
|
||||
|
||||
Key properties:
|
||||
| Property | Default | Description |
|
||||
|---|---|---|
|
||||
| `roomName` | `""` | Room to join |
|
||||
| `urlParameterName` | `"room"` | URL param for room name (`?room=xyz`) |
|
||||
| `joinRandomRoom` | `undefined` | Join random room if no name set |
|
||||
| `autoRejoin` | `true` | Auto-reconnect on disconnect |
|
||||
| `requireRoomParameter` | `false` | Only join if URL has room param |
|
||||
| `createJoinButton` | `true` | Show join/leave button in menu |
|
||||
|
||||
---
|
||||
|
||||
## @syncField (auto-sync fields)
|
||||
```ts
|
||||
@syncField() score: number = 0; // auto-syncs on reassignment
|
||||
|
||||
// With change callback:
|
||||
@syncField(MyClass.prototype.onHealthChange)
|
||||
health: number = 100;
|
||||
|
||||
private onHealthChange(newVal: number, oldVal: number) {
|
||||
console.log(`Health changed: ${oldVal} → ${newVal}`);
|
||||
}
|
||||
|
||||
// Complex types — must reassign to trigger sync:
|
||||
@syncField() items: string[] = [];
|
||||
this.items.push("sword");
|
||||
this.items = this.items; // ← triggers sync
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## SyncedTransform (sync position/rotation)
|
||||
Syncs an object's position, rotation, and scale across clients. Ownership is automatic — when a user interacts (e.g. via DragControls), they take ownership. **Always prefer `SyncedTransform` over manually sending position via custom events** — it handles interpolation, ownership, and late-joiner state automatically.
|
||||
|
||||
```ts
|
||||
import { SyncedTransform } from "@needle-tools/engine";
|
||||
|
||||
// Add to ANY Object3D — cameras, player objects, scene objects, anything
|
||||
myObject.addComponent(SyncedTransform);
|
||||
|
||||
// IMPORTANT: SyncedTransform only sends updates if you have ownership.
|
||||
// Request ownership before modifying the transform:
|
||||
const sync = myObject.getComponent(SyncedTransform);
|
||||
sync?.requestOwnership(); // fire-and-forget (ownership arrives async ~100ms)
|
||||
myObject.worldPosition = newPos; // may not broadcast immediately — ownership is async
|
||||
|
||||
// For interactive objects (e.g. DragControls), ownership is taken automatically on interaction.
|
||||
```
|
||||
|
||||
**Critical: add SyncedTransform to the prefab BEFORE networking, not after spawning.**
|
||||
`SyncedTransform` uses its component guid to match state across clients. When added via `addComponent` independently on each client, each gets a random guid — they'll never match. Add it to the prefab before `syncInstantiate` or `PlayerSync.setupFrom` so the seeded `InstantiateIdProvider` generates matching deterministic guids on all clients.
|
||||
|
||||
```ts
|
||||
// CORRECT — add SyncedTransform to the prefab before networking
|
||||
const avatarPrefab = ObjectUtils.createPrimitive("Sphere");
|
||||
avatarPrefab.guid = "player-avatar";
|
||||
avatarPrefab.addComponent(SyncedTransform); // part of the prefab — guid will be deterministic
|
||||
|
||||
const ps = await PlayerSync.setupFrom(avatarPrefab);
|
||||
ctx.scene.add(ps.gameObject);
|
||||
|
||||
// onPlayerSpawned only fires for the LOCAL player's avatar.
|
||||
// To detect ALL players (local + remote), use PlayerState.OwnerChanged:
|
||||
PlayerState.addEventListener(PlayerStateEvent.OwnerChanged, (evt) => {
|
||||
const { playerState } = evt.detail;
|
||||
const avatar = playerState.gameObject;
|
||||
if (playerState.isLocalPlayer) {
|
||||
avatar.visible = false; // hide own avatar (we see through the camera)
|
||||
avatar.getComponent(SyncedTransform)?.requestOwnership();
|
||||
} else {
|
||||
// Remote player — color/customize their avatar
|
||||
}
|
||||
});
|
||||
|
||||
// WRONG — adding SyncedTransform after spawn gives each client a random component guid
|
||||
// avatar.addComponent(SyncedTransform); // DON'T DO THIS — guids won't match
|
||||
```
|
||||
|
||||
**Timing:** Set up `PlayerSync` (add to scene) **before** `SyncedRoom` connects. If `SyncedRoom` joins a room before `PlayerSync` is enabled, the join events fire before `PlayerSync` is listening — `onPlayerSpawned` will never be called. Either add `PlayerSync` to the scene first, or set up `SyncedRoom` after `PlayerSync` is ready.
|
||||
|
||||
Do NOT manually replicate position with `connection.send("player-position", { x, y, z })` — use `SyncedTransform` instead. It uses efficient binary messages (flatbuffers) rather than JSON, making it much faster for high-frequency transform updates. Custom events are for gameplay data (scores, actions, chat), not for transform replication.
|
||||
|
||||
---
|
||||
|
||||
## PlayerSync + PlayerState (player avatar management)
|
||||
`PlayerSync` instantiates a prefab for each player joining a room and destroys it on leave. The prefab must have a `PlayerState` component. This is the recommended approach for multiplayer player objects and WebXR avatars.
|
||||
|
||||
```ts
|
||||
import { PlayerSync, PlayerState } from "@needle-tools/engine";
|
||||
|
||||
// Runtime setup — load a GLB as the player prefab:
|
||||
const ps = await PlayerSync.setupFrom("assets/avatar.glb");
|
||||
scene.add(ps.gameObject);
|
||||
// The GLB should have a PlayerState component. setupFrom() adds one if missing.
|
||||
|
||||
// Events:
|
||||
ps.onPlayerSpawned // EventList<Object3D> — fires when any player instance spawns
|
||||
```
|
||||
|
||||
**PlayerState** — attached to each spawned player instance:
|
||||
```ts
|
||||
// Static helpers:
|
||||
PlayerState.isLocalPlayer(obj) // true if obj belongs to this client
|
||||
PlayerState.all // all PlayerState instances in the scene
|
||||
PlayerState.local // only local player's PlayerState instances
|
||||
PlayerState.getFor(obj) // find PlayerState for an Object3D or Component
|
||||
|
||||
// Instance:
|
||||
state.isLocalPlayer // boolean
|
||||
state.owner // connection ID of the owning player
|
||||
|
||||
// Events:
|
||||
PlayerState.addEventListener(PlayerStateEvent.OwnerChanged, (evt) => {
|
||||
// evt.detail: { playerState, oldValue, newValue }
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Voice & Video: Voip and ScreenCapture
|
||||
Both require an active networked room and HTTPS.
|
||||
|
||||
```ts
|
||||
import { Voip, ScreenCapture } from "@needle-tools/engine";
|
||||
|
||||
// Voice chat — auto-connects when joining a room
|
||||
const voip = myObject.addComponent(Voip, { autoConnect: true, createMenuButton: true });
|
||||
voip.connect(); // manual start
|
||||
voip.disconnect(); // manual stop
|
||||
voip.setMuted(true); // mute mic
|
||||
voip.volume = 0.5; // set incoming audio volume (0–1)
|
||||
|
||||
// Access raw audio elements for spatial audio / Web Audio API routing
|
||||
const audioEl = voip.getAudioElement(userId);
|
||||
if (audioEl) {
|
||||
const audioCtx = new AudioContext();
|
||||
const source = audioCtx.createMediaElementSource(audioEl);
|
||||
const panner = audioCtx.createPanner();
|
||||
source.connect(panner).connect(audioCtx.destination);
|
||||
}
|
||||
|
||||
// Speaking detection — fires when a user starts/stops speaking
|
||||
voip.onSpeakingChanged.addEventListener((evt) => {
|
||||
console.log(evt.userId, evt.isSpeaking, evt.volume); // volume: 0–1
|
||||
});
|
||||
voip.speakingThreshold = 30; // amplitude threshold (0–255, default 30)
|
||||
|
||||
// Iterate all incoming streams
|
||||
for (const [userId, audioEl] of voip.incomingStreams) { /* ... */ }
|
||||
|
||||
// Screen/camera/microphone sharing
|
||||
const sc = myObject.addComponent(ScreenCapture);
|
||||
sc.share({ device: "Screen" }); // "Screen", "Camera", "Microphone", "Canvas"
|
||||
sc.close(); // stop sharing
|
||||
// Receiving clients see the video on a VideoPlayer component on the same object
|
||||
```
|
||||
|
||||
| Voip property | Default | Description |
|
||||
|---|---|---|
|
||||
| `autoConnect` | `true` | Start when joining a room |
|
||||
| `runInBackground` | `true` | Stay connected when tab loses focus |
|
||||
| `createMenuButton` | `true` | Show mute/unmute button in menu |
|
||||
| `volume` | `1` | Incoming audio volume (0–1, applies to all streams) |
|
||||
| `speakingThreshold` | `30` | Amplitude threshold for speaking detection (0–255) |
|
||||
|
||||
---
|
||||
|
||||
### Syncing Animations
|
||||
For syncing Animator state across clients, see the [SyncedAnimator sample](https://github.com/needle-tools/needle-engine-samples/blob/main/package/Runtime/Networking/Scripts/Networking~/Animator/SyncedAnimator.ts) — it listens for Animator parameter changes and broadcasts them via `context.connection`.
|
||||
|
||||
---
|
||||
|
||||
## Typical multiplayer setup
|
||||
1. Add `SyncedRoom` to an object (or call `context.connection.joinRoom()` manually)
|
||||
2. For player avatars: add `PlayerSync` with a prefab that has `PlayerState`
|
||||
3. For synced objects: add `SyncedTransform` to movable objects
|
||||
4. For custom state: use `@syncField()` on component properties
|
||||
5. For custom events: use `context.connection.send()` / `beginListen()`
|
||||
6. For voice chat: add `Voip` — for screen sharing: add `ScreenCapture`
|
||||
97
.agents/skills/needle-engine/references/physics.md
Normal file
97
.agents/skills/needle-engine/references/physics.md
Normal file
@@ -0,0 +1,97 @@
|
||||
# Needle Engine — Physics Reference
|
||||
|
||||
Needle Engine uses Rapier (WASM) for physics. The Rapier physics backend is registered automatically at engine startup — no manual initialization needed. The WASM binary loads lazily on first use (when a collider or rigidbody component is created), so there's no upfront cost if physics aren't used.
|
||||
|
||||
`NEEDLE_ENGINE_MODULES.RAPIER_PHYSICS.load()` and `.ready()` exist for advanced use cases (e.g. accessing the raw Rapier API directly) but are **not required** for normal physics usage — just add `Rigidbody` and collider components and they work.
|
||||
|
||||
## Colliders
|
||||
|
||||
Pick the shape that best fits the object — don't default to BoxCollider for everything.
|
||||
|
||||
```ts
|
||||
import { BoxCollider, SphereCollider, CapsuleCollider, MeshCollider } from "@needle-tools/engine";
|
||||
|
||||
// Quick setup — auto-fits to mesh bounds, optionally adds rigidbody:
|
||||
BoxCollider.add(myMesh, { rigidbody: true });
|
||||
SphereCollider.add(myMesh, { rigidbody: true });
|
||||
|
||||
// Or add manually and configure:
|
||||
const box = myObject.addComponent(BoxCollider);
|
||||
// box.size, box.center
|
||||
|
||||
const sphere = myObject.addComponent(SphereCollider);
|
||||
// sphere.radius (default: 0.5), sphere.center
|
||||
|
||||
const capsule = myObject.addComponent(CapsuleCollider);
|
||||
// capsule.radius (default: 0.5), capsule.height (default: 2) — use for characters, poles, bottles
|
||||
|
||||
const mesh = myObject.addComponent(MeshCollider);
|
||||
// mesh.convex = true for dynamic objects (required with Rigidbody)
|
||||
// mesh.convex = false for static concave geometry (walls, terrain)
|
||||
```
|
||||
|
||||
Use `SphereCollider` for balls, `CapsuleCollider` for characters/cylinders, `MeshCollider` for complex static geometry. Set `isTrigger = true` for trigger volumes.
|
||||
|
||||
## Rigidbody
|
||||
```ts
|
||||
import { Rigidbody } from "@needle-tools/engine";
|
||||
|
||||
const rb = myObject.getComponent(Rigidbody);
|
||||
rb.useGravity = true;
|
||||
rb.mass = 2.0;
|
||||
rb.isKinematic = false; // true = not affected by forces
|
||||
|
||||
// Forces and impulses
|
||||
rb.applyForce(new Vector3(0, 10, 0)); // continuous force (acceleration, applied over time)
|
||||
rb.setForce(new Vector3(0, 10, 0)); // reset + apply new force in one call
|
||||
rb.applyImpulse(new Vector3(5, 0, 0)); // instant velocity change (use for jumps, hits, explosions)
|
||||
|
||||
// Velocity — read and write directly (ALWAYS use these instead of accessing internals)
|
||||
const vel = rb.getVelocity(); // current linear velocity (Vector3)
|
||||
rb.setVelocity(new Vector3(0, 0, 0)); // set linear velocity directly
|
||||
rb.setVelocity(0, 0, 0); // also accepts x, y, z args
|
||||
const angVel = rb.getAngularVelocity(); // current angular velocity
|
||||
rb.setAngularVelocity(new Vector3(0, 0, 0));
|
||||
rb.smoothedVelocity; // averaged over ~10 frames (useful for UI/predictions)
|
||||
|
||||
// Stopping / resetting motion
|
||||
rb.resetVelocities(); // zero out both linear and angular velocity
|
||||
rb.resetForces(); // cancel all applied forces
|
||||
rb.resetTorques(); // cancel all applied torques
|
||||
rb.resetForcesAndTorques(); // cancel both forces and torques
|
||||
|
||||
// Positioning
|
||||
rb.teleport({ x: 0, y: 5, z: 0 }); // move without physics (resets velocities/forces)
|
||||
|
||||
// Sleep state
|
||||
rb.wakeUp(); // wake a sleeping body
|
||||
rb.isSleeping; // check if body is asleep
|
||||
```
|
||||
|
||||
**Force vs Impulse:** `applyForce()` is for continuous effects (thrusters, wind) — call every frame. `applyImpulse()` is for instant one-shot velocity changes (jumps, hits, button press) — call once.
|
||||
|
||||
**Never access `rb._body` or internal Rapier handles directly.** All velocity and force control is available through the public methods above. For example, to brake a rolling ball on key release, use `rb.getVelocity()` + `rb.setVelocity()` — not `(rb as any)._body.linvel()`.
|
||||
|
||||
Key properties: `mass`, `autoMass`, `useGravity`, `gravityScale` (multiplier, 0 = no gravity), `drag` (linear damping), `angularDrag`, `isKinematic`, `lockPositionX/Y/Z`, `lockRotationX/Y/Z`, `sleepThreshold`, `dominanceGroup`, `collisionDetectionMode` (Discrete or Continuous).
|
||||
|
||||
API reference: https://engine.needle.tools/docs/api/Rigidbody
|
||||
|
||||
## Physics callbacks
|
||||
Defined on components (require a Collider on the same GameObject):
|
||||
```ts
|
||||
onCollisionEnter(col: Collision) { /* hit something */ }
|
||||
onCollisionStay(col: Collision) { /* still touching */ }
|
||||
onCollisionExit(col: Collision) { /* separated */ }
|
||||
onTriggerEnter(col: Collision) { /* entered trigger */ }
|
||||
onTriggerStay(col: Collision)
|
||||
onTriggerExit(col: Collision)
|
||||
```
|
||||
|
||||
## Raycasting
|
||||
```ts
|
||||
// Visual raycast (hits any visible geometry, no collider needed, BVH-accelerated)
|
||||
const hits = this.context.physics.raycast();
|
||||
|
||||
// Physics engine raycast (hits Rapier colliders only)
|
||||
const hit = this.context.physics.engine?.raycast(origin, direction);
|
||||
```
|
||||
111
.agents/skills/needle-engine/references/postprocessing.md
Normal file
111
.agents/skills/needle-engine/references/postprocessing.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# Needle Engine — Post-Processing Reference
|
||||
|
||||
Needle Engine uses the [pmndrs postprocessing](https://github.com/pmndrs/postprocessing) library. Postprocessing loads asynchronously via `NEEDLE_ENGINE_MODULES.POSTPROCESSING` (same pattern as physics). Add and remove effects via `this.context.postprocessing`.
|
||||
|
||||
## API (`this.context.postprocessing`)
|
||||
```ts
|
||||
import { BloomEffect } from "@needle-tools/engine";
|
||||
|
||||
// Add/remove effects
|
||||
const bloom = new BloomEffect();
|
||||
bloom.intensity.value = 3;
|
||||
bloom.threshold.value = 0.5;
|
||||
this.context.postprocessing.addEffect(bloom);
|
||||
this.context.postprocessing.removeEffect(bloom);
|
||||
|
||||
// Other API
|
||||
this.context.postprocessing.markDirty(); // force rebuild next frame
|
||||
this.context.postprocessing.effects; // readonly array of active effects
|
||||
this.context.postprocessing.multisampling = "auto"; // "auto" or number (0 to max)
|
||||
this.context.postprocessing.adaptiveResolution = true; // reduce DPR when FPS drops
|
||||
```
|
||||
|
||||
## Built-in effects
|
||||
|
||||
All imported from `@needle-tools/engine`. Properties use `VolumeParameter` — set values with `.value`:
|
||||
|
||||
```ts
|
||||
// Bloom — glow on bright areas
|
||||
const bloom = new BloomEffect();
|
||||
bloom.threshold.value = 0.9; // brightness cutoff (default: 0.9)
|
||||
bloom.intensity.value = 1; // glow strength (default: 1)
|
||||
bloom.scatter.value = 0.7; // spread (default: 0.7)
|
||||
|
||||
// Depth of Field — focus blur
|
||||
import { DepthOfField, DepthOfFieldMode } from "@needle-tools/engine";
|
||||
const dof = new DepthOfField();
|
||||
dof.mode = DepthOfFieldMode.Bokeh; // Off, Gaussian, or Bokeh
|
||||
dof.focusDistance.value = 1; // focus distance
|
||||
dof.focalLength.value = 0.2; // focus range
|
||||
dof.aperture.value = 20; // bokeh scale
|
||||
|
||||
// Vignette — darkened edges
|
||||
const vig = new Vignette();
|
||||
vig.intensity.value = 0.5; // darkness (default: 0)
|
||||
vig.color.value = { r: 0, g: 0, b: 0, a: 1 };
|
||||
|
||||
// Color Adjustments — exposure, contrast, hue, saturation
|
||||
const ca = new ColorAdjustments();
|
||||
ca.postExposure.value = 1; // exposure (default: 1)
|
||||
ca.contrast.value = 0; // -1 to 1
|
||||
ca.hueShift.value = 0; // hue rotation
|
||||
ca.saturation.value = 0; // saturation adjustment
|
||||
|
||||
// Tonemapping
|
||||
const tm = new ToneMappingEffect();
|
||||
tm.setMode("AgX"); // ACES, AgX, Neutral, etc.
|
||||
tm.exposure.value = 1;
|
||||
|
||||
// Chromatic Aberration — color fringing
|
||||
const chr = new ChromaticAberration();
|
||||
chr.intensity.value = 0.5;
|
||||
|
||||
// Pixelation
|
||||
const pix = new PixelationEffect();
|
||||
pix.granularity.value = 10; // pixel size
|
||||
|
||||
// SSAO — ambient occlusion
|
||||
const ssao = new ScreenSpaceAmbientOcclusion();
|
||||
ssao.intensity.value = 2;
|
||||
ssao.samples.value = 9; // quality vs performance
|
||||
ssao.falloff.value = 1;
|
||||
ssao.color.value = new Color(0, 0, 0);
|
||||
|
||||
// N8AO — alternative AO (higher quality)
|
||||
import { ScreenSpaceAmbientOcclusionN8, ScreenSpaceAmbientOcclusionN8QualityMode } from "@needle-tools/engine";
|
||||
const n8ao = new ScreenSpaceAmbientOcclusionN8();
|
||||
n8ao.aoRadius.value = 1; // world-space radius
|
||||
n8ao.intensity.value = 1;
|
||||
n8ao.quality = ScreenSpaceAmbientOcclusionN8QualityMode.Medium;
|
||||
|
||||
// Antialiasing (SMAA)
|
||||
const aa = new Antialiasing();
|
||||
aa.preset.value = 2; // 0=Low, 1=Medium, 2=High, 3=Ultra
|
||||
|
||||
// Tilt Shift — miniature/diorama look
|
||||
const ts = new TiltShiftEffect();
|
||||
ts.focusArea.value = 0.4; // in-focus band size
|
||||
ts.feather.value = 0.3; // blur transition
|
||||
ts.offset.value = 0; // vertical offset
|
||||
ts.rotation.value = 0; // angle
|
||||
|
||||
// Sharpening
|
||||
const sharp = new SharpeningEffect();
|
||||
sharp.amount = 1; // strength (direct property, not VolumeParameter)
|
||||
sharp.radius = 1; // radius
|
||||
```
|
||||
|
||||
## Runtime parameter changes
|
||||
```ts
|
||||
// VolumeParameter values update the underlying shader uniforms immediately
|
||||
bloom.intensity.value = 5; // takes effect next frame, no rebuild needed
|
||||
|
||||
// Enable/disable individual effects
|
||||
bloom.enabled = false; // removes from pipeline
|
||||
```
|
||||
|
||||
## Notes
|
||||
- Post-processing is disabled during XR sessions.
|
||||
- Multisampling auto-adjusts: disabled when SMAA is present, scales down on low FPS, scales up when stable.
|
||||
- Effects are automatically ordered (Bloom before Vignette before ToneMapping, etc.). Custom effects can set `order` to control placement.
|
||||
- Alpha is preserved through the pipeline.
|
||||
248
.agents/skills/needle-engine/references/troubleshooting.md
Normal file
248
.agents/skills/needle-engine/references/troubleshooting.md
Normal file
@@ -0,0 +1,248 @@
|
||||
# Needle Engine — Troubleshooting
|
||||
|
||||
## Component Not Instantiated from GLB
|
||||
|
||||
**Symptom:** Component exists in Unity/Blender scene but `getComponent(MyComponent)` returns null at runtime.
|
||||
|
||||
**Causes & fixes:**
|
||||
1. **Missing `@registerType`** — Every component class must have `@registerType` above the class declaration. Without it the GLB deserializer can't match the class name to the serialized data.
|
||||
2. **Class not imported** — The file containing the class must be imported somewhere in your entry point (`main.ts`). Tree-shaking can eliminate unreferenced classes.
|
||||
3. **Name mismatch** — The C# class name in Unity must exactly match the TypeScript class name. Check for typos.
|
||||
4. **Wrong namespace** — If the Unity C# class is in a namespace, the TypeScript class must match (or the codegen mapping must be set up).
|
||||
5. **Name duplicates** — If multiple classes have the same name, the deserializer may pick the wrong one. Ensure unique class names for components.
|
||||
|
||||
```ts
|
||||
// ✅ Correct
|
||||
@registerType
|
||||
export class MyComponent extends Behaviour { ... }
|
||||
|
||||
// ❌ Wrong — missing @registerType
|
||||
export class MyComponent extends Behaviour { ... }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Decorators Not Working / Fields Always Undefined
|
||||
|
||||
**Symptom:** `@serializable` fields are always their default TypeScript values; deserialized values never appear.
|
||||
|
||||
**Fix:** Check `tsconfig.json`:
|
||||
```json
|
||||
{
|
||||
"compilerOptions": {
|
||||
"experimentalDecorators": true,
|
||||
"useDefineForClassFields": false // ← CRITICAL — must be false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`useDefineForClassFields: true` (the TS5+ default) causes class field initializers to run *after* decorators, overwriting deserialized values.
|
||||
|
||||
---
|
||||
|
||||
## GLB Not Loading / Scene Is Empty
|
||||
|
||||
**Checklist:**
|
||||
1. Is the `src` path on `<needle-engine>` correct? Paths are relative to the HTML file.
|
||||
2. Is the file in `assets/` (not `src/` or `public/`)? Static assets belong in `assets/` for Vite to copy them.
|
||||
3. Check browser console for 404 errors on the GLB request.
|
||||
4. If the file exists but scene is empty: check if the root object is active in Unity before export.
|
||||
5. CORS issues when loading from a different origin — serve from the same host or configure CORS headers.
|
||||
|
||||
---
|
||||
|
||||
## `@syncField` Not Syncing
|
||||
|
||||
**Symptom:** Field changes locally but other clients don't see updates.
|
||||
|
||||
**Causes:**
|
||||
1. **No `SyncedRoom`** in the scene — networking requires a `SyncedRoom` component or a component that connects to a room via `this.context.connection` API
|
||||
2. **Mutating array/object in place** — `this.arr.push(x)` does NOT trigger sync. You must reassign: `this.arr = [...this.arr, x]` or `this.arr = this.arr`.
|
||||
3. **Missing `@registerType`** on the component — sync relies on class registration.
|
||||
4. **Not connected** — check `this.context.connection.isConnected`.
|
||||
|
||||
---
|
||||
|
||||
## Physics Callbacks Never Fire
|
||||
|
||||
**Symptom:** `onCollisionEnter`, `onTriggerEnter`, etc. never called.
|
||||
|
||||
**Requirements:**
|
||||
- Rapier physics must be active — add a `Rigidbody` or `Collider` component in Unity on both objects
|
||||
- The GameObject must have a `Collider` component (Box, Sphere, Mesh, etc.)
|
||||
- For trigger events, the collider must be set to **Is Trigger** in Unity
|
||||
- Both objects need collider components — mesh-only objects don't participate in physics events
|
||||
|
||||
---
|
||||
|
||||
## `onDestroy` Not Called When Removing Component
|
||||
|
||||
**By design:** `removeComponent(comp)` detaches the component from update loops but does **not** call `onDestroy`. Think of it as detaching without cleanup.
|
||||
|
||||
**Fix:** Use `destroy(myComponent)` to fully clean up an object and all its components. If you need cleanup on component removal specifically, call `destroy` manually before `removeComponent()`.
|
||||
|
||||
---
|
||||
|
||||
## Animation Not Playing
|
||||
|
||||
**Checklist:**
|
||||
1. `Animator` component must be on the same or parent GameObject
|
||||
2. State name must match exactly what's in the AnimatorController
|
||||
3. Check that `animator.runtimeAnimatorController` is set (not null)
|
||||
4. If calling `play()` in `awake()`, try `start()` instead — the animator may not be initialized yet
|
||||
|
||||
---
|
||||
|
||||
## Vite Build Fails with Decorator Errors
|
||||
|
||||
Typical error: `Experimental support for decorators is a feature that is subject to change`
|
||||
|
||||
**Fix:** Ensure `tsconfig.json` has:
|
||||
```json
|
||||
"experimentalDecorators": true
|
||||
```
|
||||
|
||||
And verify that `vite.config.ts` uses the Needle plugins (they configure esbuild/swc for decorator support automatically):
|
||||
```ts
|
||||
import { needlePlugins } from "@needle-tools/engine/vite";
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## TypeScript Errors on `this.context` or `this.gameObject`
|
||||
|
||||
**Symptom:** TS error: Property 'context' does not exist on type 'MyComponent'
|
||||
|
||||
**Fix:** Make sure you extend `Behaviour` or `Component` (not a plain class):
|
||||
```ts
|
||||
import { Behaviour } from "@needle-tools/engine";
|
||||
export class MyComponent extends Behaviour { ... }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## XR Session Doesn't Start
|
||||
|
||||
**Checklist:**
|
||||
1. Must be served over **HTTPS** (or localhost) — WebXR is blocked on plain HTTP
|
||||
2. `WebXR` component must be in the scene (added in Unity or created in TS)
|
||||
3. Device must support WebXR — test with [WebXR Emulator](https://chrome.google.com/webstore/detail/webxr-api-emulator) in Chrome
|
||||
4. Check browser console for XR-related permission errors
|
||||
|
||||
---
|
||||
|
||||
## Performance: Frame Rate Drop
|
||||
|
||||
**Common causes:**
|
||||
- Per-frame `new Vector3()` / `new THREE.Color()` allocations — reuse objects
|
||||
- `getComponent()` called every frame — cache the result in `start()`
|
||||
- `findObjectOfType()` called every frame — very slow, use `start()` or events
|
||||
- Too many draw calls — use instancing or merge geometries in Unity before export
|
||||
- Large uncompressed textures — enable **Texture Compression** in Unity Needle settings
|
||||
|
||||
```ts
|
||||
// ❌ Bad — allocates every frame
|
||||
update() {
|
||||
const pos = new Vector3(1, 0, 0);
|
||||
this.gameObject.position.copy(pos);
|
||||
}
|
||||
|
||||
// ✅ Good — reuse
|
||||
private _pos = new Vector3(1, 0, 0);
|
||||
update() {
|
||||
this.gameObject.position.copy(this._pos);
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Node.js Required
|
||||
|
||||
Needle Engine projects require **Node.js** to be installed. If `npm` commands fail or Vite doesn't start, verify Node.js is installed (`node -v`). LTS version recommended.
|
||||
|
||||
---
|
||||
|
||||
## Inspecting a Running Scene (Needle Inspector Agent)
|
||||
|
||||
The [Needle Inspector](https://inspector.needle.tools/) lets you inspect a running Three.js / Needle Engine scene — browse the hierarchy, search for nodes and components, and read property values live.
|
||||
|
||||
**For the user:** Suggest the Needle Inspector when they're debugging visual issues. There's also a Chrome Extension for a visual UI.
|
||||
|
||||
**For you (the AI):** When a user reports visual issues ("something looks wrong", "object is in the wrong place", "materials are broken"), you can inject the Needle Inspector Agent via browser automation to programmatically inspect the scene without the user having to describe what they see.
|
||||
|
||||
**Inject into the running page:**
|
||||
```js
|
||||
await page.addScriptTag({ url: 'https://inspector.needle.tools/agent.js' });
|
||||
await page.waitForFunction(() => window.__NEEDLE_INSPECTOR__?.ready);
|
||||
```
|
||||
|
||||
**Use the API to inspect** (all methods return Promises via `page.evaluate`):
|
||||
```js
|
||||
const inspector = window.__NEEDLE_INSPECTOR__;
|
||||
|
||||
// Scene hierarchy and search
|
||||
await inspector.getHierarchy(); // full scene tree (depth: 10)
|
||||
await inspector.searchNodes("Player"); // find nodes by name
|
||||
|
||||
// Read properties
|
||||
await inspector.getProperties(nodeId); // all properties of a node
|
||||
await inspector.readProperty(nodeId, "position.x"); // specific value
|
||||
|
||||
// Find components
|
||||
await inspector.callTool("component_search", { regex: "Rigidbody" });
|
||||
```
|
||||
|
||||
Full tool schema: https://inspector.needle.tools/agent.md
|
||||
|
||||
---
|
||||
|
||||
## Reading Runtime Logs (Dev Server)
|
||||
|
||||
During development, Needle Engine's vite plugin automatically captures browser console output and writes it to disk. **When a user is playtesting and reports an issue, read these log files instead of asking them to copy-paste console output.**
|
||||
|
||||
**Log location:** `node_modules/.needle/logs/`
|
||||
|
||||
**File naming:** `<TIMESTAMP>.<PROCESS>.needle.log`
|
||||
- `server` — vite dev server output
|
||||
- `client` — browser console logs (log, warn, error, debug) forwarded via WebSocket
|
||||
|
||||
```bash
|
||||
# Read the most recent client log
|
||||
ls -t node_modules/.needle/logs/*.client.needle.log | head -1 | xargs cat
|
||||
```
|
||||
|
||||
The client log includes:
|
||||
- All `console.log/warn/error` calls from the browser
|
||||
- Device info (resolution, GPU, memory) logged on page load
|
||||
- Unhandled errors and promise rejections
|
||||
- Page lifecycle events (visibility, focus, navigation)
|
||||
|
||||
Logs are auto-rotated (last 30 files kept). Logging is disabled when browser DevTools are open (use `?needle-debug` URL param to force it).
|
||||
|
||||
---
|
||||
|
||||
## Build Info (`needle.buildinfo.json`)
|
||||
|
||||
After `npm run build`, a `needle.buildinfo.json` file is written to the `dist/` folder. It's also included in Needle Cloud deployments. Read it to understand the build output:
|
||||
|
||||
```json
|
||||
{
|
||||
"time": "2026-04-07T12:34:56.000Z",
|
||||
"totalsize": 5242880,
|
||||
"files": [
|
||||
{ "path": "assets/scene.glb", "hash": "abc123...", "size": 3145728 },
|
||||
{ "path": "index.html", "hash": "def456...", "size": 1024 }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Useful for: checking total build size, verifying assets are included, comparing builds (via file hashes), debugging missing files in deployments.
|
||||
|
||||
---
|
||||
|
||||
## Getting More Help
|
||||
|
||||
- Search docs: `needle_search("your question here")`
|
||||
- [Needle Engine Docs](https://engine.needle.tools/docs/)
|
||||
- [Community Forum](https://forum.needle.tools)
|
||||
- [Discord](https://discord.needle.tools)
|
||||
403
.agents/skills/needle-engine/references/xr.md
Normal file
403
.agents/skills/needle-engine/references/xr.md
Normal file
@@ -0,0 +1,403 @@
|
||||
# Needle Engine — WebXR Reference
|
||||
|
||||
## Table of Contents
|
||||
- [Overview](#overview)
|
||||
- [Starting XR Sessions](#starting-xr-sessions)
|
||||
- [XRRig and Movement](#xrrig-and-movement)
|
||||
- [Component XR Lifecycle](#component-xr-lifecycle)
|
||||
- [NeedleXRController](#needlexrcontroller)
|
||||
- [Pointer Events in XR](#pointer-events-in-xr)
|
||||
- [XR + Networking (Avatars)](#xr--networking-avatars)
|
||||
- [Image Tracking](#image-tracking)
|
||||
- [Depth Sensing](#depth-sensing)
|
||||
- [DOM Overlay (HTML in AR)](#dom-overlay-html-in-ar)
|
||||
- [iOS AR (USDZ + App Clip)](#ios-ar-usdz--app-clip)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Needle Engine supports WebXR for both VR and AR experiences. XR works across:
|
||||
- **VR headsets** (Meta Quest, etc.) — `immersive-vr` mode
|
||||
- **AR on Android** (Chrome) — `immersive-ar` mode via WebXR
|
||||
- **AR on iOS** — via USDZ Quick Look export, or via the Needle App Clip (which provides real WebXR AR on iOS)
|
||||
|
||||
The `WebXR` component (added in Unity/Blender) handles the AR/VR buttons and session setup automatically. From code, use `NeedleXRSession.start()`.
|
||||
|
||||
---
|
||||
|
||||
## Starting XR Sessions
|
||||
|
||||
```ts
|
||||
import { NeedleXRSession } from "@needle-tools/engine";
|
||||
|
||||
// Start VR
|
||||
await NeedleXRSession.start("immersive-vr");
|
||||
|
||||
// Start AR via WebXR (Android, Quest, etc.)
|
||||
await NeedleXRSession.start("immersive-ar");
|
||||
|
||||
// Shorthand "ar" — WebXR AR on supported devices, USDZ Quick Look on iOS
|
||||
await NeedleXRSession.start("ar");
|
||||
|
||||
// With custom session init (e.g. request additional features)
|
||||
await NeedleXRSession.start("immersive-ar", {
|
||||
optionalFeatures: ["camera-access", "plane-detection", "mesh-detection"]
|
||||
});
|
||||
|
||||
// Check XR state anytime
|
||||
this.context.xr?.isInXR // boolean
|
||||
this.context.xr?.session // XRSession
|
||||
this.context.xr?.mode // "immersive-vr" | "immersive-ar"
|
||||
this.context.xr?.controllers // NeedleXRController[]
|
||||
```
|
||||
|
||||
### Default features requested by Needle Engine
|
||||
|
||||
These are requested automatically — you don't need to add them:
|
||||
|
||||
**AR (`immersive-ar`):** `anchors`, `local-floor`, `layers`, `dom-overlay`, `hit-test`, `unbounded`, `hand-tracking` (except on visionOS)
|
||||
|
||||
**VR (`immersive-vr`):** `local-floor`, `bounded-floor`, `high-fixed-foveation-level`, `layers`, `hand-tracking` (except on visionOS)
|
||||
|
||||
**Not requested by default** — add these via `onBeforeXR` or the session init if you need them:
|
||||
- `camera-access` — needed for AR screenshots/camera feed compositing (add `ARCameraBackground` component or request manually)
|
||||
- `depth-sensing` — depth-based occlusion
|
||||
- `plane-detection` — detect real-world planes
|
||||
- `mesh-detection` — detect room mesh geometry
|
||||
- `image-tracking` — track reference images (added automatically by `WebXRImageTracking` component)
|
||||
|
||||
```ts
|
||||
// Add extra features via component lifecycle:
|
||||
onBeforeXR(mode: XRSessionMode, init: XRSessionInit) {
|
||||
if (mode === "immersive-ar") {
|
||||
init.optionalFeatures ??= [];
|
||||
init.optionalFeatures.push("camera-access", "plane-detection");
|
||||
}
|
||||
}
|
||||
|
||||
// Or via static event (global scope, outside components):
|
||||
NeedleXRSession.onSessionRequestStart(evt => {
|
||||
evt.init.optionalFeatures?.push("mesh-detection");
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## XRRig and Movement
|
||||
|
||||
The `XRRig` component defines the player's position and scale in XR. It's the parent transform for the headset and controllers — moving/rotating the XRRig moves the player in the scene. If no XRRig exists, one is created automatically.
|
||||
|
||||
```ts
|
||||
import { XRRig } from "@needle-tools/engine";
|
||||
// Add to an object in Unity/Blender, or create from code
|
||||
// The rig's world position = the player's feet position
|
||||
// The rig's scale controls the player's size relative to the scene
|
||||
// In AR: a larger rig scale makes the scene appear smaller (you're "bigger" relative to it)
|
||||
```
|
||||
|
||||
### XRControllerMovement
|
||||
Built-in locomotion: thumbstick movement + snap/smooth turn + teleport.
|
||||
```ts
|
||||
import { XRControllerMovement } from "@needle-tools/engine";
|
||||
// Add to an object in the scene — works automatically with XRRig
|
||||
// movementSpeed: 1.5 (m/s), rotationType: snap or smooth, teleport: enabled by default
|
||||
```
|
||||
|
||||
### TeleportTarget
|
||||
Mark surfaces as valid teleport destinations.
|
||||
```ts
|
||||
import { TeleportTarget } from "@needle-tools/engine";
|
||||
// Add to floor/ground objects — XRControllerMovement uses these as valid teleport targets
|
||||
```
|
||||
|
||||
You can create custom XR movement by implementing `onUpdateXR` on your own component, or by extending the built-in XR components:
|
||||
|
||||
```ts
|
||||
import { Behaviour, NeedleXREventArgs, NeedleXRController, registerType } from "@needle-tools/engine";
|
||||
import { Vector3 } from "three";
|
||||
|
||||
@registerType
|
||||
export class MyXRMovement extends Behaviour {
|
||||
speed = 2;
|
||||
|
||||
onUpdateXR(args: NeedleXREventArgs) {
|
||||
const rig = args.xr.rig;
|
||||
if (!rig) return;
|
||||
|
||||
// Move with left thumbstick
|
||||
for (const ctrl of args.xr.controllers) {
|
||||
const stick = ctrl.getStick("xr-standard-thumbstick");
|
||||
if (stick && (Math.abs(stick.x) > 0.1 || Math.abs(stick.y) > 0.1)) {
|
||||
// Get forward/right from the controller ray direction
|
||||
const forward = new Vector3(0, 0, -1).applyQuaternion(ctrl.rayWorldQuaternion);
|
||||
forward.y = 0;
|
||||
forward.normalize();
|
||||
const right = new Vector3(1, 0, 0).applyQuaternion(ctrl.rayWorldQuaternion);
|
||||
right.y = 0;
|
||||
right.normalize();
|
||||
|
||||
const dt = this.context.time.deltaTime;
|
||||
rig.gameObject.position.add(forward.multiplyScalar(-stick.y * this.speed * dt));
|
||||
rig.gameObject.position.add(right.multiplyScalar(stick.x * this.speed * dt));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Component XR Lifecycle
|
||||
|
||||
Implement these optional methods on any component extending `Behaviour`:
|
||||
|
||||
```ts
|
||||
import { Behaviour, NeedleXREventArgs, NeedleXRControllerEventArgs, registerType } from "@needle-tools/engine";
|
||||
|
||||
@registerType
|
||||
export class MyXRComponent extends Behaviour {
|
||||
|
||||
// Filter which XR modes this component handles
|
||||
supportsXR(mode: XRSessionMode): boolean { return true; }
|
||||
|
||||
// Modify session init params before the session starts
|
||||
onBeforeXR(mode: XRSessionMode, args: XRSessionInit) {
|
||||
args.optionalFeatures?.push("hand-tracking");
|
||||
}
|
||||
|
||||
onEnterXR(args: NeedleXREventArgs) {
|
||||
console.log("Entered XR, mode:", args.xr.mode);
|
||||
// args.xr is the NeedleXRSession
|
||||
}
|
||||
|
||||
onUpdateXR(args: NeedleXREventArgs) {
|
||||
// Per-frame during XR — access controllers here
|
||||
for (const ctrl of args.xr.controllers) {
|
||||
const pos = ctrl.gripWorldPosition;
|
||||
const rot = ctrl.gripWorldQuaternion;
|
||||
}
|
||||
}
|
||||
|
||||
onLeaveXR(args: NeedleXREventArgs) {
|
||||
console.log("Left XR");
|
||||
}
|
||||
|
||||
onXRControllerAdded(args: NeedleXRControllerEventArgs) {
|
||||
console.log("Controller added:", args.controller.index, args.controller.isHand ? "hand" : "controller");
|
||||
}
|
||||
|
||||
onXRControllerRemoved(args: NeedleXRControllerEventArgs) {
|
||||
console.log("Controller removed");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## NeedleXRController
|
||||
|
||||
Wraps an `XRInputSource` — either a physical controller or a hand. Controller inputs are also emitted as pointer events, so `onPointerDown`/`onPointerClick` on components work with controllers too.
|
||||
|
||||
```ts
|
||||
// Access in onUpdateXR or via context:
|
||||
const controllers = this.context.xr?.controllers ?? [];
|
||||
|
||||
for (const ctrl of controllers) {
|
||||
// Identity
|
||||
ctrl.index // 0 = left, 1 = right (typically)
|
||||
ctrl.isHand // true if hand tracking, false if controller
|
||||
ctrl.hand // XRHand (if hand tracking)
|
||||
ctrl.profiles // input source profiles
|
||||
ctrl.connected // still connected?
|
||||
|
||||
// Spatial data (rig space)
|
||||
ctrl.gripPosition // Vector3 — grip position in rig space
|
||||
ctrl.gripQuaternion // Quaternion — grip rotation in rig space
|
||||
ctrl.rayPosition // Vector3 — ray origin in rig space
|
||||
ctrl.rayQuaternion // Quaternion — ray direction in rig space
|
||||
|
||||
// Spatial data (world space)
|
||||
ctrl.gripWorldPosition // Vector3
|
||||
ctrl.gripWorldQuaternion // Quaternion
|
||||
ctrl.rayWorldPosition // Vector3
|
||||
ctrl.rayWorldQuaternion // Quaternion
|
||||
|
||||
// Buttons and sticks (named access)
|
||||
ctrl.getButton("trigger") // { value, pressed, touched }
|
||||
ctrl.getButton("squeeze")
|
||||
ctrl.getButton("primary-button") // A/X button
|
||||
ctrl.getStick("xr-standard-thumbstick") // { x, y }
|
||||
|
||||
// Raw gamepad
|
||||
ctrl.gamepad // Gamepad object
|
||||
|
||||
// Hit testing
|
||||
ctrl.raycastHit // current raycast result (if any)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Pointer Events in XR
|
||||
|
||||
XR controllers and hands emit pointer events through the same system as mouse/touch. Your components' `onPointerDown`, `onPointerClick`, etc. work automatically with XR input.
|
||||
|
||||
### PointerEventData
|
||||
|
||||
The `PointerEventData` passed to pointer callbacks contains:
|
||||
|
||||
```ts
|
||||
onPointerClick(args: PointerEventData) {
|
||||
// Source identification
|
||||
args.event // NEPointerEvent — the original event
|
||||
args.event.mode // "screen" (mouse/touch), "tracked-pointer" (controller), "gaze", "transient-pointer" (hand)
|
||||
args.deviceIndex // 0 for mouse/touch, controller index for XR
|
||||
args.pointerId // unique pointer+button combo ID
|
||||
args.button // 0=left, 1=middle, 2=right (mouse); button index (controller)
|
||||
args.buttonName // "LeftButton", "trigger", "squeeze", etc.
|
||||
args.pressure // 0–1 pressure value
|
||||
|
||||
// Hit information
|
||||
args.object // Object3D that was hit
|
||||
args.point // Vector3 — world position of the hit
|
||||
args.normal // Vector3 — surface normal at hit point
|
||||
args.distance // distance from origin to hit
|
||||
args.face // triangle face that was hit
|
||||
|
||||
// State
|
||||
args.isDown // true on pointer down frame
|
||||
args.isUp // true on pointer up frame
|
||||
args.isPressed // true while held
|
||||
args.isClick // true on click
|
||||
args.isDoubleClick // true on double click
|
||||
|
||||
// Control
|
||||
args.use() // mark as consumed (other handlers won't receive it)
|
||||
args.used // true if already consumed
|
||||
args.setPointerCapture() // receive move events even when pointer leaves this object
|
||||
args.releasePointerCapture()
|
||||
args.stopPropagation() // stop event from reaching other handlers
|
||||
}
|
||||
```
|
||||
|
||||
**Screen coordinates:** `args.event.clientX` / `args.event.clientY` give the screen position of the pointer (for mouse/touch). For world-to-screen projection, use Three.js standard: `worldPos.clone().project(camera)` then convert to pixels.
|
||||
|
||||
Use `args.event.mode` to distinguish between mouse, touch, and XR controllers:
|
||||
- `"screen"` — mouse or touch
|
||||
- `"tracked-pointer"` — XR controller ray
|
||||
- `"gaze"` — gaze-based input
|
||||
- `"transient-pointer"` — XR hand pinch
|
||||
|
||||
---
|
||||
|
||||
## XR + Networking (Avatars)
|
||||
|
||||
The `WebXR` component takes a reference to an avatar prefab — when a user enters XR, their avatar is spawned and synced to other users via `PlayerSync`.
|
||||
|
||||
Typical XR multiplayer setup:
|
||||
1. Add `SyncedRoom` for room management
|
||||
2. Add `WebXR` component and assign an avatar prefab (the prefab must have `PlayerState`)
|
||||
3. The avatar prefab should have `SyncedTransform` on the root and any tracked parts (head, hands)
|
||||
4. Use `PlayerState.isLocalPlayer` to distinguish between local and remote players (e.g. hide the local player's head mesh to avoid seeing it from inside)
|
||||
|
||||
The XRRig position is synced via the avatar's `SyncedTransform`. Controller/hand positions are synced as child objects of the avatar. See the [networking reference](networking.md) for full details on PlayerSync and PlayerState.
|
||||
|
||||
---
|
||||
|
||||
## Depth Sensing
|
||||
|
||||
WebXR depth sensing provides per-pixel depth information from the device's depth sensor. This enables realistic occlusion where real-world objects appear in front of virtual ones.
|
||||
|
||||
Enable via the `WebXR` component's depth sensing toggle, or request manually:
|
||||
```ts
|
||||
await NeedleXRSession.start("immersive-ar", {
|
||||
optionalFeatures: ["depth-sensing"]
|
||||
});
|
||||
```
|
||||
|
||||
Needle Engine uses the depth data automatically for occlusion when available — no additional code needed in most cases.
|
||||
|
||||
---
|
||||
|
||||
## Image Tracking
|
||||
|
||||
Track real-world images (markers) in AR sessions. Each tracked image maps a reference image to a 3D object that gets placed at the detected position. The `image-tracking` feature is automatically requested when `WebXRImageTracking` is in the scene.
|
||||
|
||||
```ts
|
||||
import { WebXRImageTracking, WebXRImageTrackingModel } from "@needle-tools/engine";
|
||||
|
||||
// Set up image tracking from code:
|
||||
const tracker = myObject.addComponent(WebXRImageTracking);
|
||||
tracker.trackedImages = [
|
||||
new WebXRImageTrackingModel({
|
||||
url: "assets/my-marker.png", // reference image URL
|
||||
widthInMeters: 0.09, // physical size of the printed marker (9cm)
|
||||
object: my3DContent, // Object3D or AssetReference to show at the marker
|
||||
imageDoesNotMove: false, // true for wall/floor markers (more stable)
|
||||
hideWhenTrackingIsLost: true, // hide when marker is no longer visible
|
||||
})
|
||||
];
|
||||
|
||||
// Listen for tracking updates:
|
||||
tracker.onTrackedImage = (images) => {
|
||||
for (const img of images) {
|
||||
console.log(img.url, img.state); // "tracked" or "emulated"
|
||||
img.applyToObject(myObj); // apply position/rotation to an object
|
||||
img.applyToObject(myObj, 0.5); // with smoothing (0–1)
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
Tips for marker images:
|
||||
- Use high-contrast images with distinct features
|
||||
- Avoid repetitive patterns or solid colors
|
||||
- `widthInMeters` must match the actual printed size — mismatched sizes cause floating/sinking
|
||||
|
||||
---
|
||||
|
||||
## DOM Overlay (HTML in AR)
|
||||
|
||||
WebXR DOM Overlay allows HTML elements to be displayed on top of the AR camera feed. Needle Engine handles this automatically — `dom-overlay` is requested by default for AR sessions.
|
||||
|
||||
During an AR session, HTML elements inside the `<needle-engine>` element are reparented into the AR overlay container so they remain visible. You can place buttons, UI, or any HTML content alongside your 3D scene, and it will appear as a 2D overlay in AR.
|
||||
|
||||
```html
|
||||
<needle-engine src="assets/scene.glb">
|
||||
<!-- These elements will be visible as overlay during AR -->
|
||||
<div style="position: absolute; bottom: 20px; left: 50%; transform: translateX(-50%);">
|
||||
<button onclick="doSomething()">My AR Button</button>
|
||||
</div>
|
||||
</needle-engine>
|
||||
```
|
||||
|
||||
Access the overlay container from code:
|
||||
```ts
|
||||
// During an AR session:
|
||||
this.context.arOverlayElement // the DOM overlay container element
|
||||
```
|
||||
|
||||
On Quest, DOM overlay is excluded (it interferes with `sessiongranted`). On Mozilla WebXR (e.g. Firefox Reality) and Needle App Clip, elements are automatically reparented to ensure visibility.
|
||||
|
||||
---
|
||||
|
||||
## iOS AR (USDZ + App Clip)
|
||||
|
||||
iOS Safari doesn't support WebXR natively. Needle Engine provides two paths:
|
||||
|
||||
**USDZ Quick Look** — Exports the scene as an interactive `.usdz` file that opens in Apple's AR viewer. Supports animations, audio, and basic interactions ("Everywhere Actions"). Configure via `USDZExporter` component or the `WebXR` component's USDZ settings.
|
||||
|
||||
**Needle App Clip (Needle Go)** — A native iOS app clip that provides real WebXR AR on iOS with full feature support (image tracking, plane detection, hand tracking). Starts automatically when the `WebXR` component is present and an iOS user taps the AR button. No extra setup needed — the App Clip loads the same web URL in a WebXR-capable native container.
|
||||
|
||||
```ts
|
||||
// Use "ar" to automatically pick the best AR path per platform:
|
||||
// Android/Quest → immersive-ar (WebXR), iOS → USDZ Quick Look or App Clip
|
||||
await NeedleXRSession.start("ar");
|
||||
|
||||
// Force USDZ Quick Look specifically:
|
||||
await NeedleXRSession.start("quicklook");
|
||||
|
||||
// immersive-ar is standard WebXR — works on Android, Quest, visionOS
|
||||
// On iOS, Needle Engine automatically launches the Needle App Clip (Needle Go) to provide WebXR support
|
||||
await NeedleXRSession.start("immersive-ar");
|
||||
```
|
||||
Reference in New Issue
Block a user