gestures

src/gesture-exploration.js

@@ -0,0 +1,471 @@
+/* potential improvements
+
+event on state switch, e.g. thumb up to thumb down or whatever to thumb down
+	but NOT thumb down to thumb down
+sustained state, e.g. thumb down to thumb down for N seconds
+extend proximityBetweenJointsCheck to any object3D or from 1 object3D to a class of entities (which themselves are object3D)
+generalize showGestureDebug to any joint, not just thumb-tip of right hand
+
+*/
+
+// consider which gestures are more inclusive, more minimalist to be more accessible
+
+targetGesture = {"microgesture":{"type":"glyph","action":"Extension","context":["Contact","Air"],"parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["thumb"]],"phalanx":[]}}
+// supports both hands
+
+otherTargetGesture = {"microgesture":{"type":"seq","quantificators":{"x":[[["thumb"],["index"],["middle"],["ring"],["pinky"]],[["thumb"],["index"],["middle"],["ring"],["pinky"]]],"y":[]},"seq":{"type":"and","operands":[{"type":"glyph","action":"Any","context":["Air","Contact"],"parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x0"]],"contact":{"type":"contact","action":"Contact","parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x1"]]},"phalanx":[]},{"type":"glyph","action":"Any","context":["Air","Contact"],"parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x1"]],"contact":{"type":"contact","action":"Contact","parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x0"]]},"phalanx":[]}]}}}
+// name = closed gap touch
+// text description = Press the specified fingers together.
+// different from "finger pinch" which is just between thumb and another finger, at specified locations
+
+// from https://lig-microglyph.imag.fr
+
+const fingersNames = ["index-finger", "middle-finger", "ring-finger", "pinky-finger","thumb"]
+const tips = fingersNames.map( f => f+"-tip" )
+const thumbParts = ["metacarpal", "phalanx-proximal", "phalanx-distal"] // no phalanx-intermediate for thumb
+const fingerParts = thumbParts.concat(["phalanx-intermediate"])
+const fingers = tips.concat( thumbParts.map( f => fingersNames.at(-1)+"-"+f ), fingerParts.flatMap( fp => fingersNames.slice(0,4).map( fn => fn+"-"+fp) ) )
+const allJointsNames = ["wrist"].concat( fingers ) // also has wrist, no fingers
+// console.log( allJointsNames.sort() )
+
+function shortVec3(v){ return {x:v.x.toFixed(3), y:v.y.toFixed(3), z:v.z.toFixed(3)} } ;
+
+// assumes joints, could be generalized to any Object3D
+function proximityBetweenJointsCheck(joints){
+	const thresholdDistance = .008
+	// contacts even while hands resting
+	// 2cm : 8
+	// 1cm : 4
+	// 9mm : 2
+	// 8mm : 0 ... but also prevents some contacts, e.g. finger tips accross fingers
+		// consequently would have to identify which contacts take place at rest
+			// might be from within same finger and thus potentially to filter out when "next" to each other joint
+				// e.g. finger tip could physiologically touch own metacarpal but no phalanx
+					// BUT it can for the same finger on the other hand
+
+	let contacts = []
+	
+	let combinations = joints.flatMap( (v, i) => joints.slice(i+1).map( w => [v, w] ))
+		// from https://stackoverflow.com/a/43241287/1442164
+
+	combinations.map( j => {
+		let rt = j[0].position
+		let lt = j[1].position
+		//console.log( 'checking: ', rt, lt )
+		let dist = rt.distanceTo(lt) 
+		if ( dist < thresholdDistance ) {
+			contacts.push( {dist: dist.toFixed(3), a:j[0].name, ah: j[0].parent.parent.el.id, b:j[1].name, bh: j[1].parent.parent.el.id } )
+			// assumes a bone, could check first on type, otherwise can have different behavior
+			// could add the timestamp and position value at that moment
+		}
+	})
+
+	return contacts
+	// getting up to 45 contacts checking 5 finger tips on each hand, which is correct for C10,2
+}
+
+// could also attach the value then show next to the joint
+let debugValue = {}
+
+function addDebbugGraph(){
+	el = document.createElement("a-box")
+	el.id = "debuggraph"
+	el.setAttribute("scale", "1 .3 .01")
+	el.setAttribute("position", "0 1.4 -1")
+	AFRAME.scenes[0].appendChild(el)
+}
+
+// used an array of points, e.g. pos.x over time, thus every 50ms xTimeSeries.push(pos.x)
+function drawPoints(points){
+	if (debugValue.length<10) return
+	let canvas = document.createElement('canvas');
+	canvas.width = 1000;
+	canvas.height = 100 * Object.values( points).length
+
+	const ctx = canvas.getContext("2d")
+	ctx.fillStyle = "white";
+	ctx.fillRect(0, 0, canvas.width, canvas.height);
+	// might want to append (and thus track status) in in order to show the result live
+		// or "just" take the last 10 elements of array
+		// middle should be 0... as we can go negative on that axis
+	//points.slice(-10).map( (p,n) => {
+	let verticalOffsetSize = 50
+	Object.values( points).map( (v,i) => {
+		ctx.beginPath()
+		ctx.moveTo(0, 0)
+		let values = v
+		if (v.length > 100) values = v.slice(-100)
+		ctx.strokeStyle = "black";
+		values.map( (p,n) => {
+			let value = Math.floor( 100-1+p*100 )
+			ctx.lineTo(n*10, value+i*verticalOffsetSize)
+			ctx.moveTo(n*10, value+i*verticalOffsetSize)
+			if (value>100-10 && value<100+10) {
+				console.log('customgesture', value)
+				AFRAME.scenes[0].emit('customgesture')
+				ctx.strokeStyle = "green";
+			}
+			
+		})
+		ctx.stroke()
+	})
+	ctx.beginPath()
+	ctx.moveTo(0, 100-10)
+	ctx.lineTo(canvas.width, 100-10)
+	ctx.moveTo(0, 100+10)
+	ctx.lineTo(canvas.width, 100+10)
+	ctx.strokeStyle = "red";
+	ctx.stroke()
+	let el = document.getElementById("debuggraph")
+	el.setAttribute("src", canvas.toDataURL() ) // somehow works on other canvas...
+	// el.object3D.children[0].material.needsUpdate = true
+	//console.log( el.src ) // works but does not update
+	return el
+}
+
+// gesture(s) to detect, callback on detection, by default debugging via console or new debuggraph
+	// should also emit event
+
+	// might need some form of hierarchy, so that composable gestures don't overlap or re-use components from other gestures
+		// e.g. if there is a fingertip to metacarpal gesture for the index, don't re-write the same for the little finger
+			// same for other hand, i.e. left vs right
+		// could be computationally interesting too, e.g. stop checking for a gesture if part of it failed already, e.g. stop checking for a gesture if part of it failed already
+
+
+AFRAME.registerComponent('cube-pull', {
+  init: function () {
+	this.tick = AFRAME.utils.throttleTick(this.tick, 50, this);
+  },
+  tick: function (t, dt) {
+	const myScene = AFRAME.scenes[0].object3D
+	if (!myScene.getObjectByName("l_handMeshNode") ) return
+	const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist")
+	let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z) 
+	console.log( sum )
+	if ( sum < .3 ) cubetest.setAttribute("position", AFRAME.utils.coordinates.stringify( wrist.position ) ) // doesn't look good, cube on wrist is moving quite a bit too
+		// could emit event too
+	// could check if all joints have close to 0 rotation on ...
+		// are roughly on the same y-plane of the wrist (facing up or down)
+   }
+})
+
+AFRAME.registerComponent('verticalflatpalmrighthand', {
+  schema: {
+  	command: {type: 'string'},
+  },
+  init: function () {
+	this.tick = AFRAME.utils.throttleTick(this.tick, 50, this);
+	window.lastGesture = Date.now() // to initialize
+  },
+  tick: function (t, dt) {
+	const myScene = AFRAME.scenes[0].object3D
+	if (!myScene.getObjectByName("l_handMeshNode") ) return
+	const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist")
+	let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z) 
+	console.log( sum )
+	//document.querySelector('[isoterminal]').emit("send", [sum + (window.lastGesture+1000>now)]) // +"\n"]) // problematic for special commands
+		//document.querySelector('[isoterminal]').emit("send", ["\n"]) // kind of horizontal
+	if ( sum < .3 ) {
+		// spams the terminal
+		// could emit event too
+		let now = Date.now()
+		if ( now - window.lastGesture > 1000 ){
+			document.querySelector('[isoterminal]').emit("send", [this.data.command]) // +"\n"]) // problematic for special commands
+			window.lastGesture = now
+			AFRAME.scenes[0].emit('gesture', {date:now, type:'vertical flat palm', value:sum})
+		}
+		// no event if inhibited due to refractory period
+	}
+	if ( sum > 1 ) {
+		let now = Date.now()
+		if ( now - window.lastGesture > 1000 ){
+			document.querySelector('[isoterminal]').emit("send", ["\n"]) // kind of horizontal
+			window.lastGesture = now
+			AFRAME.scenes[0].emit('gesture', {date:now, type:'horizontal flat palm', value:sum})
+		}
+		// no event if inhibited due to refractory period
+	}
+	// could check if all joints have close to 0 rotation on ...
+		// are roughly on the same y-plane of the wrist (facing up or down)
+   }
+})
+
+AFRAME.registerComponent('gestures', {
+  init: function () {
+	this.tick = AFRAME.utils.throttleTick(this.tick, 50, this);
+	// maybe every 50ms is too often, or not often enough, to check
+  },
+  // consider also tock, cf https://aframe.io/docs/1.7.0/core/component.html#before-after
+  tick: function (t, dt) {
+	// consider here the order or hierarchy
+		// optimisation? reconciliation? cascading?
+
+	// example of context, e.g. using  distance of wrist to a target object, or time, or any other condition
+
+	// could be defined else as long as it respect a specific format, like filters/converters/etc
+		// starting with emiting an event, e.g.
+		// AFRAME.scenes[0].emit('gesture', {date:now, type:'horizontal flat palm', value:sum})
+		// or context (unsure if truly has to be separated)
+		// AFRAME.scenes[0].emit('gesturecontext', {date:now, type:'right wrist in volume of target' })
+
+	// detect gesture A
+		// emit event
+	// detect gesture B
+		// emit event
+	// detect gesture C
+		// emit event
+  },
+})
+
+AFRAME.registerComponent('gesture-listneres', {
+	events: {
+		gesture : function (event) {
+			console.log( 'gesture captured:', event.detail )
+		},
+		gesturecontext : function (event) {
+			console.log( 'gesture context captured:', event.detail )
+		}
+	}
+})
+
+AFRAME.registerComponent('positional-context', {
+  schema: {
+    target: {type: 'selector'},
+    attribute: {type: 'string'},
+    value: {type: 'string'},
+    threshold: {type: 'float'}
+  },
+  init: function () {
+	this.tick = AFRAME.utils.throttleTick(this.tick, 500, this);
+  },
+  tick: function (t, dt) {
+	const myScene = AFRAME.scenes[0].object3D
+	if (!myScene.getObjectByName("l_handMeshNode") ) return
+	const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist").position
+	// console.log ( this.data.target.object3D.position, this.data.threshold, wrist.distanceTo( this.data.target.object3D.position ) < this.data.threshold )
+	if ( wrist.distanceTo( this.data.target.object3D.position ) < this.data.threshold ){
+		// could emit event too
+		AFRAME.scenes[0].setAttribute(this.data.attribute, "command:"+ this.data.value)
+		// wrong... this should still delegate instead to a gesture (or gestures) that would THEN conditionally pass the command
+		AFRAME.scenes[0].emit('gesturecontext', {date:now, type:'right wrist in volume of target' })
+		// could be used to check for enter/leave based on previous state until now
+	} else {
+		AFRAME.scenes[0].removeAttribute(this.data.attribute)
+	}
+   }
+})
+
+setTimeout( _ => {
+	// username based to isolate testing
+        if (username && username == "cubepull") {
+		AFRAME.scenes[0].setAttribute("cube-pull", "")
+	}
+	// multi-users as multi-gestures?
+
+	// cascading gestures
+
+	console.log ( window.location, window.location.search.includes("xrsh") )
+	//if ( window.location.search.includes("xrsh") ) {
+	if ( window.location.host.includes("fabien.benetou.fr") ) {
+	// URL modified by XRSH
+        // if (username && username == "xrsh") {
+		let el = document.createElement("a-sphere")
+		el.id = "contextsphere"
+		el.setAttribute("radius", "0.3") // fails on .3 or 0.3
+		//el.setAttribute("scale", "0.3)
+		el.setAttribute("wireframe", "true")
+		el.setAttribute("position", "0 1.4 -1")
+		AFRAME.scenes[0].appendChild(el)
+
+console.log( 1337 )
+		let cube = document.createElement("a-box")
+                cube.setAttribute("target", "") // no effect, probably over written by XRSH
+                cube.setAttribute("color", "green")
+                cube.id = "cubetest"
+		cube.setAttribute("scale", ".1 .1 .1")
+		cube.setAttribute("position", "0 1.4 -1")
+		AFRAME.scenes[0].appendChild(cube)
+
+		AFRAME.scenes[0].setAttribute("positional-context", "target:#contextsphere; threshold:0.3; attribute:verticalflatpalmrighthand; value: ls -l\\n;")
+
+		AFRAME.scenes[0].setAttribute('gesture-listneres','')
+		// gesture to 
+			// go back in history (then down again)
+			// to execute
+		/*
+			// enter
+			document.querySelector('[isoterminal]').emit("send", ["\n"])
+
+			// arrow keyup
+			document.querySelector('[isoterminal]').emit("send", ["\x1b[A"])
+
+			// arrow keydown
+			document.querySelector('[isoterminal]').emit("send", ["\x1b[B"]) 
+		*/
+
+	}
+		// that can also be visible, e.g. wireframe
+        if (username && username == "cubepullwithincontext") {
+
+                let cube = addCubeWithAnimations()
+                cube.setAttribute("target", "")
+                //cube.setAttribute("visible", "false")
+
+		el = document.createElement("a-sphere")
+		el.id = "contextsphere"
+		el.setAttribute("radius", .3)
+		el.setAttribute("wireframe", "true")
+		el.setAttribute("position", "0 1.4 -1")
+		AFRAME.scenes[0].appendChild(el)
+
+		AFRAME.scenes[0].setAttribute("positional-context", "target:#contextsphere; threshold:0.3; attribute:cube-pull;")
+	}
+		// that can also be visible, e.g. wireframe
+
+		// could do so with example of index finger tip within range of targets, if so make a visual change
+			// on within cube, off if outside of it
+				// could change the cube color to show the difference too
+
+	// try perf on Quest1
+}, 1000 )
+
+// should be a component instead...
+setTimeout( _ => {
+	const myScene = AFRAME.scenes[0].object3D
+	/*
+	setInterval( i => {
+		if (!myScene.getObjectByName("l_handMeshNode") ) return
+		const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist")
+		let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z) 
+		console.log( sum )
+		if ( sum < .3 ) cubetest.setAttribute("position") = wrist.position // doesn't look good, cube on wrist is moving quite a bit too
+		// could check if all joints have close to 0 rotation on ...
+			// are roughly on the same y-plane of the wrist (facing up or down)
+	}, 500 )
+	*/
+	/*
+	gestureThumbEndingAnyContact = setInterval( i => {
+			if (!myScene.getObjectByName("l_handMeshNode") ) return
+			// potential shortcuts :
+			const leftHandJoints = myScene.getObjectByName("l_handMeshNode").parent.children.filter( e => e.type == "Bone")
+			const rightHandJoints = myScene.getObjectByName("r_handMeshNode").parent.children.filter( e => e.type == "Bone")
+			const allHandsJoints = leftHandJoints.concat( rightHandJoints )
+
+			let posA = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position
+			let contactPointsToThumbA = leftHandJoints
+			.concat( rightHandJoints[0].parent.children.filter( e => e.name != 'thumb-tip' ) ) // right hand except thumb tip
+			.map( e => e.position.distanceTo(posA) ).filter( d => d < .02 ) // threshold of 2cm distance (must be bigger than thumb-tip to previous join position)
+											// relatively compact description and maybe relatively computively cheap
+			let pos = myScene.getObjectByName("l_handMeshNode").parent.getObjectByName("thumb-tip").position
+			let contactPointsToThumb = rightHandJoints
+			.concat( leftHandJoints[0].parent.children.filter( e => e.name != 'thumb-tip' ) ) // right hand except thumb tip
+			.map( e => e.position.distanceTo(pos) ).filter( d => d < .02 ) // threshold of 2cm distance (must be bigger than thumb-tip to previous join position)
+			if (contactPointsToThumb.length+contactPointsToThumbA.length < 1) console.log('no contact'); else console.log('thumb tip in contact with same hand or other hand')
+			// on contact could also return the join number/names
+	}, 500 )
+	*/
+
+	/*
+	testAvegageValue = setInterval( i => {
+		if (!myScene.getObjectByName("r_handMeshNode") ) return
+		let rt = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip");
+		debugValue.x.push( rt.position.x )
+		let v = debugValue.x
+		const windowSize = 10 // otherwise too long, e.g 100x500ms gives 5s average
+		if (v.length > windowSize) {
+			values = v.slice(-windowSize)
+			let avg  = ( values.reduce( (acc,c) => acc+c )/windowSize) .toFixed(3)
+			console.log( avg )
+		}
+	}, 50 )
+	*/
+
+	/*
+	showContactPoints = setInterval( i => {
+
+		if (!myScene.getObjectByName("r_handMeshNode") ) return
+		let targetJoints = []
+		tips.map( t => targetJoints.push( myScene.getObjectByName("r_handMeshNode").parent.getObjectByName(t) ) )
+		tips.map( t => targetJoints.push( myScene.getObjectByName("l_handMeshNode").parent.getObjectByName(t) ) )
+			// tips only
+		let contacts = proximityBetweenJointsCheck(targetJoints) 
+		if (contacts.length) {
+			console.log( "contacts:", contacts )
+			// {dist: dist.toFixed(3), a:j[0].name, ah: j[0].parent.parent.el.id, b:j[1].name, bh: j[1].parent.parent.el.id }
+			contacts.map( c => {
+				// show value or even just a temporary object there
+				let a = document.getElementById(c.ah).object3D.getObjectByName(c.a)
+				let b = document.getElementById(c.bh).object3D.getObjectByName(c.b)
+				const geometry = new THREE.BoxGeometry( .01, .01, .01 )
+				const material = new THREE.MeshBasicMaterial( { color: 0x00ff00 } )
+				const cube = new THREE.Mesh( geometry, material )
+				a.add( cube )
+
+			})
+		}
+	})
+	*/
+
+	/*
+	showGestureDistanceDebugJoints = setInterval( i => {
+
+		if (!myScene.getObjectByName("r_handMeshNode") ) return
+		let targetJoints = []
+		tips.map( t => targetJoints.push( myScene.getObjectByName("r_handMeshNode").parent.getObjectByName(t) ) )
+		tips.map( t => targetJoints.push( myScene.getObjectByName("l_handMeshNode").parent.getObjectByName(t) ) )
+			// console.log( targetJoints ) looks fine
+		//console.log( "contacts:", proximityBetweenJointsCheck(targetJoints)
+			// tips only
+		let targetJointsFull = []
+		allJointsNames.map( t => targetJointsFull.push( myScene.getObjectByName("r_handMeshNode").parent.getObjectByName(t) ) )
+		allJointsNames.map( t => targetJointsFull.push( myScene.getObjectByName("l_handMeshNode").parent.getObjectByName(t) ) )
+		let contacts = proximityBetweenJointsCheck(targetJointsFull) 
+		if (contacts.length) console.log( "contacts:", contacts )
+	})
+	*/
+
+	/*
+	showGestureDistanceDebug = setInterval( i => {
+		if (!myScene.getObjectByName("r_handMeshNode") ) return
+		let rt = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position
+		let lt = myScene.getObjectByName("l_handMeshNode").parent.getObjectByName("thumb-tip").position
+		if ( rt.distanceTo(lt) < .1 )
+			console.log( 'lt close to rt')
+		else
+			console.log( rt.distanceTo(lt) ) 
+	})
+	*/
+
+	/*
+	showGestureDebug = setInterval( i => {
+		if (!myScene.getObjectByName("r_handMeshNode") ) return
+		let rt = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip");
+		//console.log( shortVec3( rt.position ), shortVec3( rt.rotation ) )
+		// could do for the 2x25 values... but then becomes unreadible, hence why showing sparklines could help
+			// can be done on HUD
+
+		if (!debugValue.x){
+			debugValue.x = []
+			debugValue.y = []
+			debugValue.z = []
+
+			debugValue.a = []
+			debugValue.b = []
+			debugValue.c = []
+		}
+
+		debugValue.x.push( rt.position.x )
+		debugValue.y.push( rt.position.y )
+		debugValue.z.push( rt.position.z )
+
+		debugValue.a.push( rt.rotation.x )
+		debugValue.b.push( rt.rotation.y )
+		debugValue.c.push( rt.rotation.z )
+
+		let el = document.getElementById("debuggraph")
+		if (!el) addDebbugGraph()
+		drawPoints( debugValue )
+	}, 50 )
+	*/
+}, 1000)
+// waiting for the scene to be loaded, could be component proper too...

src/gestures_tests.html

@@ -0,0 +1,480 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>xrsh v0.143</title>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
+    <link rel="stylesheet" href="./index.css"></link>
+    <script src="./assets/aframe.min.js"></script>
+
+    <script src="com/require.js"></script>
+    <script src="com/isoterminal.js"></script>
+    
+    <!-- todo fix lazy initialisation of require.js -->
+    <script src="com/launcher.js"></script>
+    <script src="com/helloworld-window.js"></script>
+    <!-- 
+	conflict but doesn't prevent the rest from executing
+    <script src="com/pressable.js"></script>
+    -->
+
+    <script src="https://cdn.jsdelivr.net/npm/aframe-troika-text/dist/aframe-troika-text.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/gh/kylebakerio/a-console@1.0.2/a-console.js"></script>
+    <script src="https://fabien.benetou.fr/pub/home/future_of_text_demo/engine/jxr-core_branch_teleport_alt_rot.js?version=cachebusing123455"></script>
+    <script src="https://fabien.benetou.fr/pub/home/future_of_text_demo/engine/jxr-postitnote.js"></script>
+    <script src="gesture-exploration.js"></script>
+
+<script>
+
+/*
+	gesture manager :
+
+	eventRecipient (if not specified document? scene?)
+	eventName
+	functionName(parameters?) // event details expected?
+
+	examples
+
+		on pinch from any hand console.log( event details)
+*/
+
+const urlParams = new URLSearchParams(window.location.search);
+const username = urlParams.get('username');
+
+localStorage.setItem("restorestate","true")
+
+AFRAME.registerComponent('useraddednote', {
+  events: {
+        useraddednote:  function (e) {
+                let noteEl = e.detail.element
+                if ( noteEl.getAttribute("value").startsWith("jxr ") ) return
+                noteEl.classList.add("manuscriptnote")
+                // dirty mix of threejs and AFrame...
+                noteEl.object3D.parent = manuscript.object3D;
+                //noteEl.setAttribute("position", manuscript.children[0].getAttribute("position") )
+                //noteEl.setAttribute("rotation", manuscript.children[0].getAttribute("rotation") )
+                setTimeout( _ => noteEl.object3D.position.set( -.4, .4 + - document.querySelectorAll(".manuscriptnote").length/10, .51), 100 )
+                // messes up direct picking after, so could do an interaction filter on pick for this class
+                        // relatively complex to keep track of but should work
+
+                // could until then prevent picking, e.g. removing the target
+        },
+  }
+})
+
+function sceneIdsToElementForRings(){
+        // does not actually traverse the scene, only gets the top level entities
+        Array.from( document.querySelector("a-scene").childNodes ).filter( el => el.id && !el.id.startsWith("note_") ).reverse().map( (el,i) => {
+                let noteEl = addNewNote("#"+el.id, "1 "+(1+i/50)+" -1")
+                noteEl.setAttribute("onpicked","startRingCheck()")
+                noteEl.setAttribute("onreleased","endRingCheck()")
+        })
+}
+
+let ringCheckInterval = null
+let ringAdded = false
+function startRingCheck(){
+        // document.querySelector("a-console").setAttribute("visible", true)
+        let pos = new THREE.Vector3()
+        let el = selectedElements.at(-1)?.element
+        if (!el) return
+
+        let historyPositions = []
+        document.querySelector("a-console").setAttribute("visible", "true")
+        ringCheckInterval = setInterval( el => {
+                el = selectedElements.at(-1)?.element
+                el.object3D.getWorldPosition( pos )
+
+		/*
+                // trailing line
+                if ( AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode") ){
+                        historyPositions.push( AFRAME.utils.coordinates.stringify( AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position ) )
+                        if (historyPositions.length > 10){
+                                let path = historyPositions.slice(-10).join(", ")
+                                console.log( path )
+                                rings.querySelector("a-tube").setAttribute("path", path)
+                                console.log( rings.querySelector("a-tube") )
+                        }
+                }
+		*/
+
+                Array.from( active_rings.children ).map( r => {
+                        r.setAttribute("animation", "property: rotation.z; from: 0; to: 360; dur: 1000; startEvents:startAnimation;")
+
+                        let pos_ring = new THREE.Vector3()
+                        r.object3D.getWorldPosition( pos_ring )
+
+                        let d = pos.distanceTo( pos_ring )
+
+                        if (d<.2) {
+                                // visual feedback on proximity
+                                r.setAttribute("wireframe", "true")
+                        } else {
+                                r.setAttribute("wireframe", "false")
+                        }
+
+                        if (d<.1) {
+                                // should append as clone of r to rings_stack
+
+                                // ding sound
+                                // audio.play()
+
+                                // restart every time
+                                r.emit('startAnimation', null, false)
+
+                                let elValue = el.getAttribute("value")
+                                // to clean up otherwise always expect a value, e.g. on gltf-model
+
+                                // ---------------------------------------- resolving -----------------------------------------------------
+                                let resolvedElements = []
+
+                                if ( elValue && elValue.length > 0 && elValue.startsWith(".") || elValue.startsWith("#") || elValue.startsWith("a-") ){
+                                        if ( elValue.startsWith(".") ){
+                                                resolvedElements = Array.from( AFRAME.scenes[0].querySelectorAll(elValue) )
+                                        } else {
+                                                el = document.querySelector(elValue)
+                                                // assuming "a-..." is unique, e.g. a-sky, which isn't true for others, e.g. a-troika-text
+                                        }
+                                }
+
+                                let value = r.querySelector("a-troika-text").getAttribute("value")
+                                // should instead "just" apply the jxr, done here just for testing
+                                        // probably need some refractory period too, otherwise stacking transformations that are probably not required
+
+                                // ---------------------------------------- this should be based on value itself, i.e. another eval() -----------------------------------------------------
+                                        // see applyFunctionToSelection() ... which does not exist
+                                                // can be replaced by something more generic, there are already jxr shortcuts e.g. sa attributeName attributeValue
+                                                // could be done as filters are done, i.e. filters.map( f => f(element) )
+
+                                resolvedElements.map( rEl => {
+                                        if (value.includes("scaleUp") ) rEl.setAttribute("scale", ".01 .02 .02")
+                                        if (value.includes("White") ) rEl.setAttribute("color", "white")
+                                        if (value.includes("Blue") ) rEl.setAttribute("color", "blue")
+                                        if (value.includes("Red") ) rEl.setAttribute("color", "red")
+                                })
+
+                                // gets busy very quickly, surely because of the lack of refractory period TODO fix
+                                if (false && value.length>1) {
+                                        console.log('should clone ', r, ' on stack', rings_stack )
+                                        let clonedRing = r.cloneNode(true)
+                                        rings_stack.appendChild( clonedRing )
+                                }
+
+                                if (value.includes("scaleUp") ) el.setAttribute("scale", ".02 .02 .02")
+                                if (value.includes("White") ) el.setAttribute("color", "white")
+                                if (value.includes("Blue") ) el.setAttribute("color", "blue")
+                                if (value.includes("Red") ) el.setAttribute("color", "red")
+
+				// assume we are ready to send command (which is often NOT true as it takes quite some time to get the ISO running, even with save)
+				let temporaryBufferForPipping = ''
+					// TODO need properly escaping/encapsulating
+				if (value.includes('isoterminal("ls")') ) document.querySelector('[isoterminal]').emit("exec", ["echo 'hello world'", (str,buf) => { console.log(str); temporaryBufferForPipping = str } ])
+				if (value.includes('isoterminal(" | wc -l")') ) document.querySelector('[isoterminal]').emit("exec", [ "echo " + temporaryBufferForPipping + " | wc", (str,buf) => console.log(str, " (temporaryBufferForPipping:"+ temporaryBufferForPipping + ")") ])
+						// temporaryBufferForPipping is empty
+					// could be tried outside of XR
+					// problem too, as here we're getting wc 1 0 1 rather than 1 2 12
+                                //if (value.includes('isoterminal(" | wc -l")') ) document.querySelector('[isoterminal]').emit("exec", [ "echo " + temporaryBufferForPipping + " | wc -l", (str,buf) => console.log(str) ])
+					// newline problem
+
+                                // could include resolving too, e.g .collidable doesn't get appended as-is but rather its resulting elements do
+                                        // could be better to do it later on, at the transformation level, going through ring
+                                if (value.includes("addToRingSelection") ) {
+                                        // try to get the selection value and append to it, if none make new one
+                                        const selectionNodeId = "selectionnote"
+                                        let selectionNoteEl = document.getElementById( selectionNodeId )
+                                        if (!selectionNoteEl){
+                                                let newPos = r.getAttribute("position").clone()
+                                                newPos.y += .3
+                                                selectionNoteEl = addNewNote( elValue, AFRAME.utils.coordinates.stringify( newPos ), "0.1 0.1 0.1", selectionNodeId) // text only, not 3D model, for this would try to get its ID instead
+                                                selectionNoteEl.setAttribute("onpicked", "startRingCheck()")
+                                                selectionNoteEl.setAttribute("onreleased", "endRingCheck()")
+                                        } else {
+                                                selectionNoteEl.setAttribute("value", selectionNoteEl.getAttribute("value") + "\n" + elValue )
+                                        }
+                                }
+
+                                if (value.length==1) el.setAttribute("value", el.getAttribute("value")+value)
+                                        // character append
+                                                // could then make a keyboard this way...
+                                                        // but 26 characters make for very large movements
+                                                                // assuming a meta-ring but the layout itself can be totally different
+                                                                // can try much smaller rings
+                                                // also need a refractory period
+                                                        // add another ring to
+                                                                // create new word (how?)
+                                                                // split existing word on " "
+
+                                // ---------------------------------------- -----------------------------------------------------
+
+                                // should then
+                                        // clone ring then append it to rings_stack (for follow up macro)
+                                                // this should be permanent, i.e. JSON WebDAV save, and movable, i.e add target attribute (done)
+                                        // show/unhide/unfold the "following" rings based on context
+
+                                // special rings ideas
+
+                                // key aspect of UI/UX
+                                if (!ringAdded){
+                                        // here we are considering a sequential "positive" flow
+                                                // but could also be negative, a la ! or conditional e.g. && or ||
+                                                        // so rings could have a type or return value equivalent
+
+                                        ringAdded = true
+                                        // need refractory period or maximum amount of times added
+                                        let newRing = addRing("jxr console.log('Red')", r.getAttribute("position") )
+                                        newRing.setAttribute("color", "red")
+                                        setTimeout( _ => newRing.object3D.position.z += .3, 100 )
+                                        // offset from current ring position
+
+                                        // this itself should enable the creation of a 4th ring, etc
+                                }
+                        }
+                })
+        }, 100)
+}
+
+function addKeyboardRings(){
+        // addRing(code, position)
+        "abcdefghijklmnopqrstuvwxyz ,.{}()[]".split('').reverse().map( (c,i) => {
+                let x = (i%3)/10 +.5
+                let y = 1+(i/3)/10
+                addRing(c, x+" "+y+ " -.5", .03, .005, .005)
+        } )
+        // could probably be done once they repositioned instead via a well known parent with id (or unique selector)
+}
+
+function addRing(code, position="0 1.5 -.7", radius=.1, radiusTubular=.01, codeVerticalOffset=.15){
+        let el = document.createElement("a-torus")
+        el.setAttribute("position", position)
+        el.setAttribute("segments-radial", "4")
+        el.setAttribute("segments-tubular", "12")
+        el.setAttribute("opacity", ".3")
+        el.setAttribute("radius", radius)
+        el.setAttribute("radius-tubular", radiusTubular)
+        let elCode = document.createElement("a-troika-text")
+        // consider binding an existing jxr command to an existing ring to have a similar behavior
+                // e.g. drop jxr on ring, bind them so that next time an element goes through, it's applied via binded ring
+        elCode.setAttribute("anchor", "left")
+        elCode.setAttribute("target", "")
+        elCode.setAttribute("value", code)
+        elCode.setAttribute("position", ".0 "+codeVerticalOffset+" .0")
+        elCode.setAttribute("scale", "0.1 0.1 0.1")
+        el.appendChild( elCode )
+        active_rings.appendChild( el )
+        return el
+}
+
+function endRingCheck(){
+        clearInterval(ringCheckInterval)
+}
+
+function applyNextFilterInteraction (){}
+let sequentialFiltersInteractionOnPicked = []
+let sequentialFiltersInteractionOnReleased = []
+
+let currentFilterOnPicked = null
+let currentFilterOnReleased = null
+
+</script>
+
+
+  </head>
+  <body style="background:linear-gradient( 45deg, #324, #8c74ff)">
+    <a-scene selfcontainer
+             light="defaultLightsEnabled: true"
+             cursor="rayOrigin: mouse"
+             renderer="colorManagement: false; stencil: true; antialias:true; highRefreshRate:true; foveationLevel: 0.5;"
+             xr-mode-ui="XRMode: xr"
+             obb-collider="showColliders:false"
+             raycaster="objects: [html]; interval:100;"
+             >
+
+	     <a-sphere id=contextsphere wireframe=true position="0 1.4 -1" radius=.3></a-sphere>
+
+	     <a-console position="-1 1.3 0" rotation="-45 90 0" font-size="34" height="1" skip-intro="true"></a-console>
+
+
+      <a-entity isoterminal="iso: https://forgejo.isvery.ninja/assets/xrsh-buildroot/main/xrsh.iso; minimized:true; overlayfs: /package.overlayfs.zip; bootMenu: 1; bootMenuURL: 1" position="0 1.6 -0.3">
+        <a-entity launcher pressable position="0 -0.315 0"></a-entity>
+      </a-entity>
+
+      <a-sphere position="1 1 -2"></a-sphere>
+      <a-entity helloworld-window pressable position="0 1.6 0"></a-entity> 
+
+
+      <a-entity id="rig">
+
+	      <a-entity id="player">
+		<a-entity camera="fov:90" position="0 1.6 0" id="camera" wasd-controls gaze-touch-to-click></a-entity>
+		<a-entity id="left-hand"  hand-tracking-grab-controls="hand: left; modelColor: #EEEEEE; hoverEnabled:true" 
+					  laser-controls="hand: left" 
+					  raycaster="far:0.04" 
+					  blink-controls="cameraRig:#player; teleportOrigin: #camera; collisionEntities: #floor"
+			pinchsecondary
+					  > 
+		</a-entity>
+		<a-entity id="right-hand" hand-tracking-grab-controls="hand: right; modelColor: #EEEEEE; hoverEnabled:true"
+					  laser-controls="hand: right"
+					  raycaster="far:0.04"
+			pinchprimary
+					  blink-controls="cameraRig:#player; teleportOrigin: #camera; collisionEntities: #floor"
+					  ____pinch-to-teleport="rig: #player"></a-entity>
+	      </a-entity>
+      </a-entity>
+
+      <a-sphere segments-width=12 segments-height=12 pressable="" start-on-press="" id="box" radius="0.033" color="gray"></a-sphere>
+      <a-box pressable="" start-on-press-other="" id="otherbox" scale=".05 .05 .05" opacity=.3 wireframe=false color="white"></a-box>
+
+
+
+        <a-entity id=rings>
+
+                      <!-- consider applyNextFilterInteraction() -->
+                <!-- sequence of rings, namely no "absolute" list, in addition to apply a function to the content currently selected, it opens up the "next" rings and stacks itself to the applied rings -->
+
+                <a-troika-text anchor=left target value= "jxr addKeyboardRings()" position="0 1.10 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text anchor=left target value= 'jxr Array.from( active_rings.querySelectorAll("a-torus") ).map( el => {  el.setAttribute("radius", ".01");  el.setAttribute("radius-tubular", ".001") })' position="0.4 1.20 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value= "jxr sceneIdsToElementForRings()" position="0 1.20 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='example text' position="0 1.30 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='a-sky' position="0 1.40 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='.collidable' position="0 1.60 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position="0 1.50 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-entity id=active_rings>
+                        <a-torus color="green" position="0.4 1.4 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr isoterminal("ls")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                        <a-torus color="green" position="0.4 1.4 -.8" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr isoterminal(" | wc -l")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                        <a-torus color="blue" position="0 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                        <a-torus color="white" position="0 1 -.3" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToWhite")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+
+                        <a-torus color="white" position=".5 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr scaleUp()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+
+                        <a-torus color="white" position="-.5 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr addRingFromCode()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+
+                        <a-torus color="white" position="-.5 1.5 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr addToRingSelection()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                </a-entity>
+
+                <!-- see history of transformations via voice, and cancelling it, older PoC -->
+                <a-entity id=rings_stack target position="-1 1 -1.5">
+                        <a-troika-text anchor=left target value='previous transformations' position="-.1 .3 .1" scale="0.1 0.1 0.1"></a-troika-text>
+                        <a-torus wireframe=true color="#43A367" position="0 0 -.1" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text curve-radius=1 rotation="90 0 0" anchor=left target value='jxr applyFunctionToSelection("changeColorToRed")' position=".0 .1 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                        <a-torus wireframe=true color="#43A367" position="0 0 0" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                </a-entity>
+
+      </a-entity>
+
+    </a-scene>
+
+  </body>
+
+  <script>
+
+	  const term=document.querySelector('[isoterminal]');
+	  term.emit("exec", ["ls -l", (str,buf) => console.log(str) ])
+
+    // HOWDY DEVELOPER! 
+    // for a REPL boilerplate see this codepen instead: https://codepen.io/coderofsalvation/pen/wBwRpew
+
+    document.querySelector('a-scene').addEventListener('isoterminal_init', function(e){
+
+
+	    const term=document.querySelector('[isoterminal]');
+	    term.addEventListener('ready', e => {
+		    term.emit("exec", ["ls -l", (str,buf) => console.log(str) ])
+	     })
+
+	// insert rings here instead
+
+      // example custom REPL (https://xrsh.isvery.ninja/#Custom%20REPLs)
+      const myREPL = {
+        key: "f",
+        prompt: "\r\nmyrepl> ",
+        title: (opts) => `other awesome NLnet FOSS projects ❤️`,
+        init: function( mainmenu ){
+          window.open('https://nlnet.nl/project/current.html','_blank')
+          this.send("\n\rreturning to mainmenu")
+          mainmenu()
+        },
+        keyHandler: function(ch){
+          this.send(ch) // echo keys to user
+        },
+        cmdHandler: function(cmd){
+          this.send("\r\n"+cmd) // only works after commenting mainmenu()\n")
+          this.send(myREPL.prompt)
+        }
+      }
+      ISOTerminal.prototype.boot.menu.push(myREPL)
+    })
+
+setTimeout( _ => {
+
+	let jointTestEl1 = addNewNote( "jxr document.querySelector('a-sky').setAttribute('color','purple')", "0 .03 -.05")
+	jointTestEl1.setAttribute("scale", ".05 .05 .05")
+	jointTestEl1.setAttribute("rotation", "0 -90 0")
+	jointTestEl1.setAttribute("target", "")
+	jointTestEl1.id = "jointtest1"
+	let jointTestEl2 = addNewNote( "jxr document.querySelector('a-sky').setAttribute('color','gray')", "0 .03 -.05")
+	jointTestEl2.setAttribute("scale", ".05 .05 .05")
+	jointTestEl2.setAttribute("position", "0 .03 -.05")
+	jointTestEl2.setAttribute("rotation", "0 -90 0")
+	jointTestEl2.id = "jointtest2"
+
+	AFRAME.scenes[0].setAttribute("bind-element-to-finger__test1", { hand: 'r_handMeshNode', finger: 'index-finger-tip', target: '#jointtest1' })
+	AFRAME.scenes[0].setAttribute("bind-element-to-finger__test2", { hand: 'r_handMeshNode', finger: 'thumb-tip', target: '#jointtest2' } )
+}, 1000)
+
+AFRAME.registerComponent('bind-element-to-finger', {
+  multiple: true,
+  schema: {
+        hand: {type: 'string', default: 'r_handMeshNode'},
+        finger: {type: 'string', default: 'index-finger-tip'},
+        target : {type: 'selector'},
+  },
+  init: function () {
+  },
+  tick: function (time, timeDelta) {
+        const joint = AFRAME.scenes[0].object3D.getObjectByName(this.data.hand)?.parent.getObjectByName(this.data.finger)
+        if ( joint && this.data.target.object3D.parent == AFRAME.scenes[0].object3D ) this.data.target.object3D.parent = joint
+        // unfortunately reparenting for now breaks properly picking...
+                // could check in core for whom the parent is and reparent only on release
+                        // or potentially not at all, maybe the user expects to be able to pick the instruction away from the hand
+                        // would also then need an explicit mechanism to bind back
+                                // a la wrist shortcut
+                // warning that some parenting is done "right" namely by removing the offset then become an A-Frame child with that offset
+                        // allowing thus the expected behavior in that context
+                        // consequently might have to distinguish the 2 situations
+  }
+})
+
+/* // does not work reliably, seems to be just the 1st keypress
+var keyEvent = document.createEvent('KeyboardEvent');
+keyEvent.initKeyboardEvent('keydown', true, false, null, 0, false, 0, false, 77, 0);
+window.dispatchEvent(keyEvent);
+*/
+
+  </script>
+
+</html>

src/gestures_text.html

@@ -0,0 +1,482 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>xrsh v0.143</title>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
+    <link rel="stylesheet" href="./index.css"></link>
+    <script src="./assets/aframe.min.js"></script>
+
+    <script src="com/require.js"></script>
+    <script src="com/isoterminal.js"></script>
+    
+    <!-- todo fix lazy initialisation of require.js -->
+    <script src="com/launcher.js"></script>
+    <script src="com/helloworld-window.js"></script>
+    <!-- 
+	conflict but doesn't prevent the rest from executing
+    <script src="com/pressable.js"></script>
+    -->
+
+    <script src="https://cdn.jsdelivr.net/npm/aframe-troika-text/dist/aframe-troika-text.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/gh/kylebakerio/a-console@1.0.2/a-console.js"></script>
+    <script src="https://fabien.benetou.fr/pub/home/future_of_text_demo/engine/jxr-core_branch_teleport_alt_rot.js?version=cachebusing123455"></script>
+    <script src="https://fabien.benetou.fr/pub/home/future_of_text_demo/engine/jxr-postitnote.js"></script>
+    <script src="https://companion.benetou.fr/gesture-exploration.js?version=cachebusting1234"></script>
+
+<script>
+
+/*
+	gesture manager :
+
+	eventRecipient (if not specified document? scene?)
+	eventName
+	functionName(parameters?) // event details expected?
+
+	examples
+
+		on pinch from any hand console.log( event details)
+*/
+
+const urlParams = new URLSearchParams(window.location.search);
+const username = urlParams.get('username');
+
+localStorage.setItem("restorestate","true")
+
+AFRAME.registerComponent('useraddednote', {
+  events: {
+        useraddednote:  function (e) {
+                let noteEl = e.detail.element
+                if ( noteEl.getAttribute("value").startsWith("jxr ") ) return
+                noteEl.classList.add("manuscriptnote")
+                // dirty mix of threejs and AFrame...
+                noteEl.object3D.parent = manuscript.object3D;
+                //noteEl.setAttribute("position", manuscript.children[0].getAttribute("position") )
+                //noteEl.setAttribute("rotation", manuscript.children[0].getAttribute("rotation") )
+                setTimeout( _ => noteEl.object3D.position.set( -.4, .4 + - document.querySelectorAll(".manuscriptnote").length/10, .51), 100 )
+                // messes up direct picking after, so could do an interaction filter on pick for this class
+                        // relatively complex to keep track of but should work
+
+                // could until then prevent picking, e.g. removing the target
+        },
+  }
+})
+
+function sceneIdsToElementForRings(){
+        // does not actually traverse the scene, only gets the top level entities
+        Array.from( document.querySelector("a-scene").childNodes ).filter( el => el.id && !el.id.startsWith("note_") ).reverse().map( (el,i) => {
+                let noteEl = addNewNote("#"+el.id, "1 "+(1+i/50)+" -1")
+                noteEl.setAttribute("onpicked","startRingCheck()")
+                noteEl.setAttribute("onreleased","endRingCheck()")
+        })
+}
+
+let ringCheckInterval = null
+let ringAdded = false
+function startRingCheck(){
+        // document.querySelector("a-console").setAttribute("visible", true)
+        let pos = new THREE.Vector3()
+        let el = selectedElements.at(-1)?.element
+        if (!el) return
+
+        let historyPositions = []
+        document.querySelector("a-console").setAttribute("visible", "true")
+        ringCheckInterval = setInterval( el => {
+                el = selectedElements.at(-1)?.element
+                el.object3D.getWorldPosition( pos )
+
+		/*
+                // trailing line
+                if ( AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode") ){
+                        historyPositions.push( AFRAME.utils.coordinates.stringify( AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position ) )
+                        if (historyPositions.length > 10){
+                                let path = historyPositions.slice(-10).join(", ")
+                                console.log( path )
+                                rings.querySelector("a-tube").setAttribute("path", path)
+                                console.log( rings.querySelector("a-tube") )
+                        }
+                }
+		*/
+
+                Array.from( active_rings.children ).map( r => {
+                        r.setAttribute("animation", "property: rotation.z; from: 0; to: 360; dur: 1000; startEvents:startAnimation;")
+
+                        let pos_ring = new THREE.Vector3()
+                        r.object3D.getWorldPosition( pos_ring )
+
+                        let d = pos.distanceTo( pos_ring )
+
+                        if (d<.2) {
+                                // visual feedback on proximity
+                                r.setAttribute("wireframe", "true")
+                        } else {
+                                r.setAttribute("wireframe", "false")
+                        }
+
+                        if (d<.1) {
+                                // should append as clone of r to rings_stack
+
+                                // ding sound
+                                // audio.play()
+
+                                // restart every time
+                                r.emit('startAnimation', null, false)
+
+                                let elValue = el.getAttribute("value")
+                                // to clean up otherwise always expect a value, e.g. on gltf-model
+
+                                // ---------------------------------------- resolving -----------------------------------------------------
+                                let resolvedElements = []
+
+                                if ( elValue && elValue.length > 0 && elValue.startsWith(".") || elValue.startsWith("#") || elValue.startsWith("a-") ){
+                                        if ( elValue.startsWith(".") ){
+                                                resolvedElements = Array.from( AFRAME.scenes[0].querySelectorAll(elValue) )
+                                        } else {
+                                                el = document.querySelector(elValue)
+                                                // assuming "a-..." is unique, e.g. a-sky, which isn't true for others, e.g. a-troika-text
+                                        }
+                                }
+
+                                let value = r.querySelector("a-troika-text").getAttribute("value")
+                                // should instead "just" apply the jxr, done here just for testing
+                                        // probably need some refractory period too, otherwise stacking transformations that are probably not required
+
+                                // ---------------------------------------- this should be based on value itself, i.e. another eval() -----------------------------------------------------
+                                        // see applyFunctionToSelection() ... which does not exist
+                                                // can be replaced by something more generic, there are already jxr shortcuts e.g. sa attributeName attributeValue
+                                                // could be done as filters are done, i.e. filters.map( f => f(element) )
+
+                                resolvedElements.map( rEl => {
+                                        if (value.includes("scaleUp") ) rEl.setAttribute("scale", ".01 .02 .02")
+                                        if (value.includes("White") ) rEl.setAttribute("color", "white")
+                                        if (value.includes("Blue") ) rEl.setAttribute("color", "blue")
+                                        if (value.includes("Red") ) rEl.setAttribute("color", "red")
+                                })
+
+                                // gets busy very quickly, surely because of the lack of refractory period TODO fix
+                                if (false && value.length>1) {
+                                        console.log('should clone ', r, ' on stack', rings_stack )
+                                        let clonedRing = r.cloneNode(true)
+                                        rings_stack.appendChild( clonedRing )
+                                }
+
+                                if (value.includes("scaleUp") ) el.setAttribute("scale", ".02 .02 .02")
+                                if (value.includes("White") ) el.setAttribute("color", "white")
+                                if (value.includes("Blue") ) el.setAttribute("color", "blue")
+                                if (value.includes("Red") ) el.setAttribute("color", "red")
+
+				// assume we are ready to send command (which is often NOT true as it takes quite some time to get the ISO running, even with save)
+				let temporaryBufferForPipping = ''
+					// TODO need properly escaping/encapsulating
+				if (value.includes('isoterminal("ls")') ) document.querySelector('[isoterminal]').emit("exec", ["echo 'hello world'", (str,buf) => { console.log(str); temporaryBufferForPipping = str } ])
+				if (value.includes('isoterminal(" | wc -l")') ) document.querySelector('[isoterminal]').emit("exec", [ "echo " + temporaryBufferForPipping + " | wc", (str,buf) => console.log(str, " (temporaryBufferForPipping:"+ temporaryBufferForPipping + ")") ])
+						// temporaryBufferForPipping is empty
+					// could be tried outside of XR
+					// problem too, as here we're getting wc 1 0 1 rather than 1 2 12
+                                //if (value.includes('isoterminal(" | wc -l")') ) document.querySelector('[isoterminal]').emit("exec", [ "echo " + temporaryBufferForPipping + " | wc -l", (str,buf) => console.log(str) ])
+					// newline problem
+
+                                // could include resolving too, e.g .collidable doesn't get appended as-is but rather its resulting elements do
+                                        // could be better to do it later on, at the transformation level, going through ring
+                                if (value.includes("addToRingSelection") ) {
+                                        // try to get the selection value and append to it, if none make new one
+                                        const selectionNodeId = "selectionnote"
+                                        let selectionNoteEl = document.getElementById( selectionNodeId )
+                                        if (!selectionNoteEl){
+                                                let newPos = r.getAttribute("position").clone()
+                                                newPos.y += .3
+                                                selectionNoteEl = addNewNote( elValue, AFRAME.utils.coordinates.stringify( newPos ), "0.1 0.1 0.1", selectionNodeId) // text only, not 3D model, for this would try to get its ID instead
+                                                selectionNoteEl.setAttribute("onpicked", "startRingCheck()")
+                                                selectionNoteEl.setAttribute("onreleased", "endRingCheck()")
+                                        } else {
+                                                selectionNoteEl.setAttribute("value", selectionNoteEl.getAttribute("value") + "\n" + elValue )
+                                        }
+                                }
+
+                                if (value.length==1) el.setAttribute("value", el.getAttribute("value")+value)
+                                        // character append
+                                                // could then make a keyboard this way...
+                                                        // but 26 characters make for very large movements
+                                                                // assuming a meta-ring but the layout itself can be totally different
+                                                                // can try much smaller rings
+                                                // also need a refractory period
+                                                        // add another ring to
+                                                                // create new word (how?)
+                                                                // split existing word on " "
+
+                                // ---------------------------------------- -----------------------------------------------------
+
+                                // should then
+                                        // clone ring then append it to rings_stack (for follow up macro)
+                                                // this should be permanent, i.e. JSON WebDAV save, and movable, i.e add target attribute (done)
+                                        // show/unhide/unfold the "following" rings based on context
+
+                                // special rings ideas
+
+                                // key aspect of UI/UX
+                                if (!ringAdded){
+                                        // here we are considering a sequential "positive" flow
+                                                // but could also be negative, a la ! or conditional e.g. && or ||
+                                                        // so rings could have a type or return value equivalent
+
+                                        ringAdded = true
+                                        // need refractory period or maximum amount of times added
+                                        let newRing = addRing("jxr console.log('Red')", r.getAttribute("position") )
+                                        newRing.setAttribute("color", "red")
+                                        setTimeout( _ => newRing.object3D.position.z += .3, 100 )
+                                        // offset from current ring position
+
+                                        // this itself should enable the creation of a 4th ring, etc
+                                }
+                        }
+                })
+        }, 100)
+}
+
+function addKeyboardRings(){
+        // addRing(code, position)
+        "abcdefghijklmnopqrstuvwxyz ,.{}()[]".split('').reverse().map( (c,i) => {
+                let x = (i%3)/10 +.5
+                let y = 1+(i/3)/10
+                addRing(c, x+" "+y+ " -.5", .03, .005, .005)
+        } )
+        // could probably be done once they repositioned instead via a well known parent with id (or unique selector)
+}
+
+function addRing(code, position="0 1.5 -.7", radius=.1, radiusTubular=.01, codeVerticalOffset=.15){
+        let el = document.createElement("a-torus")
+        el.setAttribute("position", position)
+        el.setAttribute("segments-radial", "4")
+        el.setAttribute("segments-tubular", "12")
+        el.setAttribute("opacity", ".3")
+        el.setAttribute("radius", radius)
+        el.setAttribute("radius-tubular", radiusTubular)
+        let elCode = document.createElement("a-troika-text")
+        // consider binding an existing jxr command to an existing ring to have a similar behavior
+                // e.g. drop jxr on ring, bind them so that next time an element goes through, it's applied via binded ring
+        elCode.setAttribute("anchor", "left")
+        elCode.setAttribute("target", "")
+        elCode.setAttribute("value", code)
+        elCode.setAttribute("position", ".0 "+codeVerticalOffset+" .0")
+        elCode.setAttribute("scale", "0.1 0.1 0.1")
+        el.appendChild( elCode )
+        active_rings.appendChild( el )
+        return el
+}
+
+function endRingCheck(){
+        clearInterval(ringCheckInterval)
+}
+
+function applyNextFilterInteraction (){}
+let sequentialFiltersInteractionOnPicked = []
+let sequentialFiltersInteractionOnReleased = []
+
+let currentFilterOnPicked = null
+let currentFilterOnReleased = null
+
+</script>
+
+
+  </head>
+  <body style="background:linear-gradient( 45deg, #324, #8c74ff)">
+    <a-scene selfcontainer
+             light="defaultLightsEnabled: true"
+             cursor="rayOrigin: mouse"
+             renderer="colorManagement: false; stencil: true; antialias:true; highRefreshRate:true; foveationLevel: 0.5;"
+             xr-mode-ui="XRMode: xr"
+             obb-collider="showColliders:false"
+             raycaster="objects: [html]; interval:100;"
+             >
+
+	     <a-sphere id=contextsphere wireframe=true position="0 1.4 -1" radius=.3></a-sphere>
+
+	     <a-console position="-1 1.3 0" rotation="-45 90 0" font-size="34" height="1" skip-intro="true"></a-console>
+
+
+      <a-entity isoterminal="iso: https://forgejo.isvery.ninja/assets/xrsh-buildroot/main/xrsh.iso; minimized:true; overlayfs: /package.overlayfs.zip; bootMenu: 1; bootMenuURL: 1" position="0 1.6 -0.3">
+        <a-entity launcher pressable position="0 -0.315 0"></a-entity>
+      </a-entity>
+
+      <a-sphere position="1 1 -2"></a-sphere>
+      <a-entity helloworld-window pressable position="0 1.6 0"></a-entity> 
+
+
+      <a-entity id="rig">
+
+	      <a-entity id="player">
+		<a-entity camera="fov:90" position="0 1.6 0" id="camera" wasd-controls gaze-touch-to-click></a-entity>
+		<a-entity id="left-hand"  hand-tracking-grab-controls="hand: left; modelColor: #EEEEEE; hoverEnabled:true" 
+					  laser-controls="hand: left" 
+					  raycaster="far:0.04" 
+					  blink-controls="cameraRig:#player; teleportOrigin: #camera; collisionEntities: #floor"
+			pinchsecondary
+					  > 
+		</a-entity>
+		<a-entity id="right-hand" hand-tracking-grab-controls="hand: right; modelColor: #EEEEEE; hoverEnabled:true"
+					  laser-controls="hand: right"
+					  raycaster="far:0.04"
+			pinchprimary
+					  blink-controls="cameraRig:#player; teleportOrigin: #camera; collisionEntities: #floor"
+					  ____pinch-to-teleport="rig: #player"></a-entity>
+	      </a-entity>
+      </a-entity>
+
+      <a-sphere segments-width=12 segments-height=12 pressable="" start-on-press="" id="box" radius="0.033" color="gray"></a-sphere>
+      <a-box pressable="" start-on-press-other="" id="otherbox" scale=".05 .05 .05" opacity=.3 wireframe=false color="white"></a-box>
+
+
+
+        <a-entity id=rings>
+
+                      <!-- consider applyNextFilterInteraction() -->
+                <!-- sequence of rings, namely no "absolute" list, in addition to apply a function to the content currently selected, it opens up the "next" rings and stacks itself to the applied rings -->
+
+                <a-troika-text anchor=left target value= "jxr addKeyboardRings()" position="0 1.10 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text anchor=left target value= 'jxr Array.from( active_rings.querySelectorAll("a-torus") ).map( el => {  el.setAttribute("radius", ".01");  el.setAttribute("radius-tubular", ".001") })' position="0.4 1.20 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value= "jxr sceneIdsToElementForRings()" position="0 1.20 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='example text' position="0 1.30 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='a-sky' position="0 1.40 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='.collidable' position="0 1.60 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position="0 1.50 -.8" scale="0.1 0.1 0.1"></a-troika-text>
+
+                <a-entity id=active_rings>
+                        <a-torus color="green" position="0.4 1.4 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr isoterminal("ls")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                        <a-torus color="green" position="0.4 1.4 -.8" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr isoterminal(" | wc -l")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                        <a-torus color="blue" position="0 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                        <a-torus color="white" position="0 1 -.3" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToWhite")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+
+                        <a-torus color="white" position=".5 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr scaleUp()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+
+                        <a-torus color="white" position="-.5 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr addRingFromCode()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+
+                        <a-torus color="white" position="-.5 1.5 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr addToRingSelection()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                </a-entity>
+
+                <!-- see history of transformations via voice, and cancelling it, older PoC -->
+                <a-entity id=rings_stack target position="-1 1 -1.5">
+                        <a-troika-text anchor=left target value='previous transformations' position="-.1 .3 .1" scale="0.1 0.1 0.1"></a-troika-text>
+                        <a-torus wireframe=true color="#43A367" position="0 0 -.1" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text curve-radius=1 rotation="90 0 0" anchor=left target value='jxr applyFunctionToSelection("changeColorToRed")' position=".0 .1 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                        <a-torus wireframe=true color="#43A367" position="0 0 0" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
+                                <a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
+                        </a-torus>
+                </a-entity>
+
+      </a-entity>
+
+    </a-scene>
+
+  </body>
+
+  <script>
+
+	  const term=document.querySelector('[isoterminal]');
+	  term.emit("exec", ["ls -l", (str,buf) => console.log(str) ])
+
+    // HOWDY DEVELOPER! 
+    // for a REPL boilerplate see this codepen instead: https://codepen.io/coderofsalvation/pen/wBwRpew
+
+    document.querySelector('a-scene').addEventListener('isoterminal_init', function(e){
+
+
+	    const term=document.querySelector('[isoterminal]');
+	    term.addEventListener('ready', e => {
+		    term.emit("exec", ["ls -l", (str,buf) => console.log(str) ])
+	     })
+
+	// insert rings here instead
+
+      // example custom REPL (https://xrsh.isvery.ninja/#Custom%20REPLs)
+      const myREPL = {
+        key: "f",
+        prompt: "\r\nmyrepl> ",
+        title: (opts) => `other awesome NLnet FOSS projects ❤️`,
+        init: function( mainmenu ){
+          window.open('https://nlnet.nl/project/current.html','_blank')
+          this.send("\n\rreturning to mainmenu")
+          mainmenu()
+        },
+        keyHandler: function(ch){
+          this.send(ch) // echo keys to user
+        },
+        cmdHandler: function(cmd){
+          this.send("\r\n"+cmd) // only works after commenting mainmenu()\n")
+          this.send(myREPL.prompt)
+        }
+      }
+      ISOTerminal.prototype.boot.menu.push(myREPL)
+    })
+
+// consider bringing https://companion.benetou.fr/gesture-exploration.js directly here instead
+
+setTimeout( _ => {
+
+	let jointTestEl1 = addNewNote( "jxr document.querySelector('a-sky').setAttribute('color','purple')", "0 .03 -.05")
+	jointTestEl1.setAttribute("scale", ".05 .05 .05")
+	jointTestEl1.setAttribute("rotation", "0 -90 0")
+	jointTestEl1.setAttribute("target", "")
+	jointTestEl1.id = "jointtest1"
+	let jointTestEl2 = addNewNote( "jxr document.querySelector('a-sky').setAttribute('color','gray')", "0 .03 -.05")
+	jointTestEl2.setAttribute("scale", ".05 .05 .05")
+	jointTestEl2.setAttribute("position", "0 .03 -.05")
+	jointTestEl2.setAttribute("rotation", "0 -90 0")
+	jointTestEl2.id = "jointtest2"
+
+	AFRAME.scenes[0].setAttribute("bind-element-to-finger__test1", { hand: 'r_handMeshNode', finger: 'index-finger-tip', target: '#jointtest1' })
+	AFRAME.scenes[0].setAttribute("bind-element-to-finger__test2", { hand: 'r_handMeshNode', finger: 'thumb-tip', target: '#jointtest2' } )
+}, 1000)
+
+AFRAME.registerComponent('bind-element-to-finger', {
+  multiple: true,
+  schema: {
+        hand: {type: 'string', default: 'r_handMeshNode'},
+        finger: {type: 'string', default: 'index-finger-tip'},
+        target : {type: 'selector'},
+  },
+  init: function () {
+  },
+  tick: function (time, timeDelta) {
+        const joint = AFRAME.scenes[0].object3D.getObjectByName(this.data.hand)?.parent.getObjectByName(this.data.finger)
+        if ( joint && this.data.target.object3D.parent == AFRAME.scenes[0].object3D ) this.data.target.object3D.parent = joint
+        // unfortunately reparenting for now breaks properly picking...
+                // could check in core for whom the parent is and reparent only on release
+                        // or potentially not at all, maybe the user expects to be able to pick the instruction away from the hand
+                        // would also then need an explicit mechanism to bind back
+                                // a la wrist shortcut
+                // warning that some parenting is done "right" namely by removing the offset then become an A-Frame child with that offset
+                        // allowing thus the expected behavior in that context
+                        // consequently might have to distinguish the 2 situations
+  }
+})
+
+/* // does not work reliably, seems to be just the 1st keypress
+var keyEvent = document.createEvent('KeyboardEvent');
+keyEvent.initKeyboardEvent('keydown', true, false, null, 0, false, 0, false, 77, 0);
+window.dispatchEvent(keyEvent);
+*/
+
+  </script>
+
+</html>