generalized gestures
Some checks are pending
/ test (push) Waiting to run

This commit is contained in:
Fabien Benetou 2025-07-17 15:23:34 +02:00
parent 265590f6d0
commit 9dcbea7eba
9 changed files with 2079 additions and 1433 deletions

229
src/drumkbd.js Normal file
View file

@ -0,0 +1,229 @@
// let sequentialFilters = []
// <script src="filters/keymap.js"></script>
let currentFilter = null
function applyNextFilter( filename ){
if ( currentFilter == null ) currentFilter = -1
currentFilter++
if ( sequentialFilters[currentFilter] ){
sequentialFilters[ currentFilter ]( filename )
} else {
console.log( "done filtering for", filename )
currentFilter = null
}
}
function showFile( filename, openingOptions = {}){
console.log('showFile', filename)
fetch( filename ).then( r => {
let idFromFilename = filename.replaceAll('.','') // has to remove from proper CSS ID
if (!filesWithMetadata[filename] ) filesWithMetadata[filename] = {}
filesWithMetadata[filename].contentType = r.headers.get('Content-Type')
filesWithMetadata[filename].idFromFilename = idFromFilename
filesWithMetadata[filename].openingOptions = openingOptions
console.log( 'ct metadata', filename, filesWithMetadata[filename].contentType )
applyNextFilter( filename )
// const showdefinitions = urlParams.get('showdefinitions');
// should be showFile() configuration parameter instead
// can be used via e.g. showFile("https://fabien.benetou.fr/?action=source",{ mereology:"whole"})
// should emit an event when done, for now just console.log which isn't programmatic
})
}
function addDrumKeyboard(){
// get keymap
showFile('fabien_corneish_zen.keymap')
// showFile('fabien_corneish_zen.keymap')
const swiping = urlParams.get('swiping');
const keyclass = "keys_from_drumsticks"
// transform keymap to keys of keyboard
let keyboardEl = document.createElement("a-entity")
keyboardEl.id = 'keyboard'
AFRAME.scenes[0].appendChild( keyboardEl )
AFRAME.scenes[0].addEventListener("keymaploaded", e => {
applyToClass("keymap_layer", el => el.setAttribute("visible", "false") )
// alternatively other layers could be displayed but with lower opacity
// might be very busy but easier to recall
Array.from( document.querySelectorAll('.keymap_layer') ).map( (layerToAdd, layerNumber) => {
layerToAdd.getAttribute("value").split('\n').map( (l,y) => {
l.split("|").map( (k,x) => {
if (k.trim()) {
// zIndex could be once deep once shallow to potentially go faster between keys, kind of straggered vs ortho
let xOffset = -.2
let yOffset = 1.3
let zOffset = -.4
let keysPerRow = l.split("|").length -1
let ratio = 1/20
zOffset -= Math.abs( keysPerRow/2 - x ) * ratio/5 // arguably more ergonomic
if (swiping){ zOffset = zOffset + Math.abs( x - keysPerRow/2 ) * ratio } // opposite from swiping
// somehow the center isn't... the center
if (l.length < 70) xOffset += .15 // 80 was fine for layer 0 but not layer 1 and 2
let labelEl = document.createElement("a-troika-text")
labelEl.setAttribute("value",k.trim())
labelEl.setAttribute("position", "0 .51 0")
labelEl.setAttribute("font-size", "1")
labelEl.setAttribute("color", "black")
labelEl.setAttribute("rotation", "-90 0 0")
let keyEl = document.createElement("a-cylinder")
keyEl.setAttribute("segments-height", "2")
keyEl.setAttribute("segments-radial", "24")
keyEl.setAttribute("scale", ".01 .01 .01")
keyEl.setAttribute("rotation", "60 0 0")
keyEl.setAttribute("position", ""+(xOffset+x*ratio)+" " +(yOffset-y*ratio)+" "+(zOffset + y/50) )
keyEl.classList.add( keyclass )
keyEl.classList.add( "layer_"+ layerNumber )
keyEl.appendChild( labelEl )
keyboardEl.appendChild( keyEl )
// keyEl.id = keyclass+'_'+k.trim() // not correct anymore as multiple layers can have the same key
keyEl.id = keyclass+'_'+layerNumber+'_'+k.trim()
// setTimeout( _ => keyEl.object3D.lookAt( new THREE.Vector3(0, 1.5, 0)), 100 )
// not great as they have 90 deg rotation already, could find a better way
// ... but arguably should be the opposite based on resting hand positions
}
})
})
})
keys_from_drumsticks_0_SHFT.setAttribute("wireframe", shiftFromVirtualKeyboard) // arguable, could do so for all layers
// hide all layers but the current one
Array.from( document.querySelectorAll(".keys_from_drumsticks") ).map( k => k.setAttribute("visible", "false") )
Array.from( document.querySelectorAll(".keys_from_drumsticks"+".layer_"+layerFromVirtualKeyboard) ).map( k => k.setAttribute("visible", "true") )
keyBoardCheck()
})
const threshold = .02 // distance
const refractionPeriod = 500 // ms until next keypress
// add visible contact points
let jointTestEl1 = document.createElement("a-sphere")
jointTestEl1.setAttribute("radius", .01)
jointTestEl1.id = "jointtest1"
AFRAME.scenes[0].appendChild( jointTestEl1 )
let jointTestEl2 = document.createElement("a-sphere")
jointTestEl2.setAttribute("radius", .01)
jointTestEl2.id = "jointtest2"
AFRAME.scenes[0].appendChild( jointTestEl2 )
AFRAME.scenes[0].setAttribute("bind-element-to-finger__test1", { hand: 'r_handMeshNode', finger: 'index-finger-tip', target: '#jointtest1' } )
AFRAME.scenes[0].setAttribute("bind-element-to-finger__test2", { hand: 'l_handMeshNode', finger: 'index-finger-tip', target: '#jointtest2' } )
// thumb tip test
let jointTestEl3 = document.createElement("a-sphere")
jointTestEl3.setAttribute("radius", .01)
jointTestEl3.id = "jointtest3"
AFRAME.scenes[0].appendChild( jointTestEl3 )
let jointTestEl4 = document.createElement("a-sphere")
jointTestEl4.setAttribute("radius", .01)
jointTestEl4.id = "jointtest4"
AFRAME.scenes[0].appendChild( jointTestEl4 )
AFRAME.scenes[0].setAttribute("bind-element-to-finger__test3", { hand: 'r_handMeshNode', finger: 'thumb-tip', target: '#jointtest3' } )
AFRAME.scenes[0].setAttribute("bind-element-to-finger__test4", { hand: 'l_handMeshNode', finger: 'thumb-tip', target: '#jointtest4' } )
const forcecontrollers = urlParams.get('forcecontrollers');
if (forcecontrollers){
setTimeout( _ => {
jointtest1.object3D.parent = document.querySelector("[meta-touch-controls]").object3D
jointtest2.object3D.parent = document.querySelector("[oculus-touch-controls]").object3D
}, 2000 )
// TODO does not work with pen
// see <a-entity logiteck-mx-ink-controls="hand: right"></a-entity> that does not seem to show anything, even in AFrame 1.7.1 nor recent master build
}
// if done a la bind-element-to-finger consider
// const joint = AFRAME.scenes[0].object3D.getObjectByName(this.data.hand)?.parent.getObjectByName(this.data.finger)
// if ( joint && this.data.target.object3D.parent == AFRAME.scenes[0].object3D ) this.data.target.object3D.parent = joint
let pos1 = new THREE.Vector3()
let pos2 = new THREE.Vector3()
let pos3 = new THREE.Vector3()
let pos4 = new THREE.Vector3()
let shiftFromVirtualKeyboard = true
let layerFromVirtualKeyboard = 0
window.keyboardTarget = typinghud
// check for potential contact
let lastKeypress = Date.now()
function keyBoardCheck() {
return setInterval( _ => {
jointTestEl1.object3D.getWorldPosition( pos1 )
jointTestEl2.object3D.getWorldPosition( pos2 )
jointTestEl3.object3D.getWorldPosition( pos3 )
jointTestEl4.object3D.getWorldPosition( pos4 )
// could also check only when jointTestEl1 / jointTestEl2 are visible, ignore otherwise
// to do with all keys instead
Array.from( document.querySelectorAll(".keys_from_drumsticks"+".layer_"+layerFromVirtualKeyboard) )
.concat( [keys_from_drumsticks_0_LWR, keys_from_drumsticks_0_RSE] )
.filter( k => k.getAttribute("visible") )
.map( k => {
// should only look at visible keys, could limit via .filter( k => k.getAttribute("visible") == "true")
// this way one wouldn't type on an invisible keyboard
let d1 = k.object3D.position.distanceTo( pos1 )
let d2 = k.object3D.position.distanceTo( pos2 )
let d3 = k.object3D.position.distanceTo( pos3 )
let d4 = k.object3D.position.distanceTo( pos4 )
if ( d1 < threshold || d2 < threshold || d3 < threshold || d4 < threshold) {
//if ( d1 < threshold || d2 < threshold) {
k.setAttribute("color", "pink")
if (keyboardTarget == typinghud) typinghud.setAttribute("material","opacity", .5)
if ( Date.now() - lastKeypress < refractionPeriod ){
// console.warn('ignoring, executed during the last 500ms already')
let x = 42 // added just to ignore
} else {
lastKeypress = Date.now()
let value = k.firstChild.getAttribute("value")
if ( keyboardTarget.getAttribute("value") == "[]" ) keyboardTarget.setAttribute("value", "" ) // for typinghud starting value
if (value == "TAB") console.warn('does not complete yet') // TODO add completion
if (value == "SPC") value = " "
if (value == "ENT") {
if (keyboardTarget == typinghud) {
parseKeys("keydown", "Enter")
keyboardTarget.setAttribute("value", "" )
} else {
keyboardTarget.setAttribute("value", keyboardTarget.getAttribute("value") + '\n' )
}
} else if (value == "SHFT") {
shiftFromVirtualKeyboard = !shiftFromVirtualKeyboard
// visual highlight, also note that is closer to CAPSLOCK behavior
keys_from_drumsticks_0_SHFT.setAttribute("wireframe", shiftFromVirtualKeyboard) // arguable, could do so for all layers
} else if (value == "RSE") {
if (layerFromVirtualKeyboard<2) layerFromVirtualKeyboard++ // hardcoded max
console.log('should raise layer', layerFromVirtualKeyboard) // a la CAPSLOCK too
Array.from( document.querySelectorAll(".keys_from_drumsticks") ).map( k => k.setAttribute("visible", "false") )
Array.from( document.querySelectorAll(".keys_from_drumsticks"+".layer_"+layerFromVirtualKeyboard) ).map( k => k.setAttribute("visible", "true") )
// forcing visibility yet get ignored as on wrong layer
keys_from_drumsticks_0_LWR.setAttribute("visible", "true")
keys_from_drumsticks_0_RSE.setAttribute("visible", "true")
} else if (value == "LWR") {
if (layerFromVirtualKeyboard>0) layerFromVirtualKeyboard--
console.log('should lower layer', layerFromVirtualKeyboard) // a la CAPSLOCK too
Array.from( document.querySelectorAll(".keys_from_drumsticks") ).map( k => k.setAttribute("visible", "false") )
Array.from( document.querySelectorAll(".keys_from_drumsticks"+".layer_"+layerFromVirtualKeyboard) ).map( k => k.setAttribute("visible", "true") )
keys_from_drumsticks_0_LWR.setAttribute("visible", "true")
keys_from_drumsticks_0_RSE.setAttribute("visible", "true")
} else if (value == "BKSP") {
keyboardTarget.setAttribute("value", keyboardTarget.getAttribute("value").slice(0,-1) )
} else {
if (!shiftFromVirtualKeyboard) value = value.toLowerCase()
keyboardTarget.setAttribute("value", keyboardTarget.getAttribute("value") + value )
}
keyboardEl.emit( "keypressed", {key: value} )
}
} else if ( d1 < threshold*1.2 || d2 < threshold*1.2) { // arguably, not convinced it brings value more than confusion
k.setAttribute("color", "#ffe4e1")
} else {
k.setAttribute("color", "white")
}
})
}, 20)
}
return keyboardEl
}

View file

@ -0,0 +1,75 @@
#include <behaviors.dtsi>
#include <dt-bindings/zmk/keys.h>
#include <dt-bindings/zmk/bt.h>
/ {
chosen {
zmk,matrix_transform = &default_transform;
//zmk,matrix_transform = &five_column_transform;
};
};
/ {
behaviors {
hm: homerow_mods {
compatible = "zmk,behavior-hold-tap";
label = "HOMEROW_MODS";
#binding-cells = <2>;
tapping-term-ms = <150>;
quick-tap-ms = <0>;
flavor = "tap-preferred";
bindings = <&kp>, <&kp>;
};
};
keymap {
compatible = "zmk,keymap";
default_layer {
label = "QWERTY";
// -----------------------------------------------------------------------------------------
// | TAB | Q | W | E | R | T | | Y | U | I | O | P | BKSP |
// | CTRL | A | S | D | F | G | | H | J | K | L | ; | ' |
// | SHFT | Z | X | C | V | B | | N | M | , | . | / | ESC |
// | GUI | LWR | SPC | | ENT | RSE | ALT |
bindings = <
&kp TAB &kp Q &kp W &kp E &kp R &kp T &kp Y &kp U &kp I &kp O &kp P &kp BSPC
&kp LCTRL &kp A &kp S &kp D &kp F &kp G &kp H &kp J &kp K &kp L &kp SEMI &kp SQT
//&kp LCTRL &kp A &kp S &kp D &kp F &kp G &kp H &kp J &kp K &kp L &kp SEMI &kp SQT
&kp LSHFT &kp Z &kp X &kp C &kp V &kp B &kp N &kp M &kp COMMA &kp DOT &kp FSLH &kp ESC
&kp LGUI &mo 1 &kp SPACE &kp RET &mo 2 &kp RALT
>;
};
lower_layer {
label = "NUMBER";
// -----------------------------------------------------------------------------------------
// | TAB | 1 | 2 | 3 | 4 | 5 | | 6 | 7 | 8 | 9 | 0 | BKSP |
// | | BT1 | BT2 | BT3 | BT4 | BT5 | | LFT | DWN | UP | RGT | | |
// | SHFT | F11 | F12 | | | | | | PAGE_DOWN | PAGE_UP | | | BTCLR |
// | GUI | | SPC | | ENT | | ALT |
bindings = <
&kp TAB &kp N1 &kp N2 &kp N3 &kp N4 &kp N5 &kp N6 &kp N7 &kp N8 &kp N9 &kp N0 &kp BSPC
&trans &bt BT_SEL 0 &bt BT_SEL 1 &bt BT_SEL 2 &bt BT_SEL 3 &bt BT_SEL 4 &kp LEFT &kp DOWN &kp UP &kp RIGHT &trans &trans
&kp LSHFT &kp F11 &kp F12 &trans &trans &trans &trans &kp PAGE_DOWN &kp PAGE_UP &trans &trans &bt BT_CLR
&kp LGUI &trans &kp SPACE &kp RET &trans &kp RALT
>;
};
raise_layer {
label = "SYMBOL";
// -----------------------------------------------------------------------------------------
// | TAB | ! | @ | # | $ | % | | ^ | & | * | ( | ) | BKSP |
// | CTRL | | | | | | | - | = | [ | ] | \ | ` |
// | SHFT | pp | vUP | vDO | | | | _ | + | { | } | "|" | ~ |
// | GUI | | SPC | | ENT | | ALT |
bindings = <
&kp TAB &kp EXCL &kp AT &kp HASH &kp DLLR &kp PRCNT &kp CARET &kp AMPS &kp KP_MULTIPLY &kp LPAR &kp RPAR &kp BSPC
&kp LCTRL &trans &trans &trans &trans &trans &kp MINUS &kp EQUAL &kp LBKT &kp RBKT &kp BSLH &kp GRAVE
&kp LSHFT &kp C_PLAY_PAUSE &kp K_VOLUME_UP &kp K_VOLUME_DOWN &trans &trans &kp UNDER &kp PLUS &kp LBRC &kp RBRC &kp PIPE &kp TILDE
&kp LGUI &trans &kp SPACE &kp RET &trans &kp RALT
>;
};
};
};

View file

@ -1,471 +0,0 @@
/* potential improvements
event on state switch, e.g. thumb up to thumb down or whatever to thumb down
but NOT thumb down to thumb down
sustained state, e.g. thumb down to thumb down for N seconds
extend proximityBetweenJointsCheck to any object3D or from 1 object3D to a class of entities (which themselves are object3D)
generalize showGestureDebug to any joint, not just thumb-tip of right hand
*/
// consider which gestures are more inclusive, more minimalist to be more accessible
targetGesture = {"microgesture":{"type":"glyph","action":"Extension","context":["Contact","Air"],"parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["thumb"]],"phalanx":[]}}
// supports both hands
otherTargetGesture = {"microgesture":{"type":"seq","quantificators":{"x":[[["thumb"],["index"],["middle"],["ring"],["pinky"]],[["thumb"],["index"],["middle"],["ring"],["pinky"]]],"y":[]},"seq":{"type":"and","operands":[{"type":"glyph","action":"Any","context":["Air","Contact"],"parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x0"]],"contact":{"type":"contact","action":"Contact","parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x1"]]},"phalanx":[]},{"type":"glyph","action":"Any","context":["Air","Contact"],"parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x1"]],"contact":{"type":"contact","action":"Contact","parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x0"]]},"phalanx":[]}]}}}
// name = closed gap touch
// text description = Press the specified fingers together.
// different from "finger pinch" which is just between thumb and another finger, at specified locations
// from https://lig-microglyph.imag.fr
const fingersNames = ["index-finger", "middle-finger", "ring-finger", "pinky-finger","thumb"]
const tips = fingersNames.map( f => f+"-tip" )
const thumbParts = ["metacarpal", "phalanx-proximal", "phalanx-distal"] // no phalanx-intermediate for thumb
const fingerParts = thumbParts.concat(["phalanx-intermediate"])
const fingers = tips.concat( thumbParts.map( f => fingersNames.at(-1)+"-"+f ), fingerParts.flatMap( fp => fingersNames.slice(0,4).map( fn => fn+"-"+fp) ) )
const allJointsNames = ["wrist"].concat( fingers ) // also has wrist, no fingers
// console.log( allJointsNames.sort() )
function shortVec3(v){ return {x:v.x.toFixed(3), y:v.y.toFixed(3), z:v.z.toFixed(3)} } ;
// assumes joints, could be generalized to any Object3D
function proximityBetweenJointsCheck(joints){
const thresholdDistance = .008
// contacts even while hands resting
// 2cm : 8
// 1cm : 4
// 9mm : 2
// 8mm : 0 ... but also prevents some contacts, e.g. finger tips accross fingers
// consequently would have to identify which contacts take place at rest
// might be from within same finger and thus potentially to filter out when "next" to each other joint
// e.g. finger tip could physiologically touch own metacarpal but no phalanx
// BUT it can for the same finger on the other hand
let contacts = []
let combinations = joints.flatMap( (v, i) => joints.slice(i+1).map( w => [v, w] ))
// from https://stackoverflow.com/a/43241287/1442164
combinations.map( j => {
let rt = j[0].position
let lt = j[1].position
//console.log( 'checking: ', rt, lt )
let dist = rt.distanceTo(lt)
if ( dist < thresholdDistance ) {
contacts.push( {dist: dist.toFixed(3), a:j[0].name, ah: j[0].parent.parent.el.id, b:j[1].name, bh: j[1].parent.parent.el.id } )
// assumes a bone, could check first on type, otherwise can have different behavior
// could add the timestamp and position value at that moment
}
})
return contacts
// getting up to 45 contacts checking 5 finger tips on each hand, which is correct for C10,2
}
// could also attach the value then show next to the joint
let debugValue = {}
function addDebbugGraph(){
el = document.createElement("a-box")
el.id = "debuggraph"
el.setAttribute("scale", "1 .3 .01")
el.setAttribute("position", "0 1.4 -1")
AFRAME.scenes[0].appendChild(el)
}
// used an array of points, e.g. pos.x over time, thus every 50ms xTimeSeries.push(pos.x)
function drawPoints(points){
if (debugValue.length<10) return
let canvas = document.createElement('canvas');
canvas.width = 1000;
canvas.height = 100 * Object.values( points).length
const ctx = canvas.getContext("2d")
ctx.fillStyle = "white";
ctx.fillRect(0, 0, canvas.width, canvas.height);
// might want to append (and thus track status) in in order to show the result live
// or "just" take the last 10 elements of array
// middle should be 0... as we can go negative on that axis
//points.slice(-10).map( (p,n) => {
let verticalOffsetSize = 50
Object.values( points).map( (v,i) => {
ctx.beginPath()
ctx.moveTo(0, 0)
let values = v
if (v.length > 100) values = v.slice(-100)
ctx.strokeStyle = "black";
values.map( (p,n) => {
let value = Math.floor( 100-1+p*100 )
ctx.lineTo(n*10, value+i*verticalOffsetSize)
ctx.moveTo(n*10, value+i*verticalOffsetSize)
if (value>100-10 && value<100+10) {
console.log('customgesture', value)
AFRAME.scenes[0].emit('customgesture')
ctx.strokeStyle = "green";
}
})
ctx.stroke()
})
ctx.beginPath()
ctx.moveTo(0, 100-10)
ctx.lineTo(canvas.width, 100-10)
ctx.moveTo(0, 100+10)
ctx.lineTo(canvas.width, 100+10)
ctx.strokeStyle = "red";
ctx.stroke()
let el = document.getElementById("debuggraph")
el.setAttribute("src", canvas.toDataURL() ) // somehow works on other canvas...
// el.object3D.children[0].material.needsUpdate = true
//console.log( el.src ) // works but does not update
return el
}
// gesture(s) to detect, callback on detection, by default debugging via console or new debuggraph
// should also emit event
// might need some form of hierarchy, so that composable gestures don't overlap or re-use components from other gestures
// e.g. if there is a fingertip to metacarpal gesture for the index, don't re-write the same for the little finger
// same for other hand, i.e. left vs right
// could be computationally interesting too, e.g. stop checking for a gesture if part of it failed already, e.g. stop checking for a gesture if part of it failed already
AFRAME.registerComponent('cube-pull', {
init: function () {
this.tick = AFRAME.utils.throttleTick(this.tick, 50, this);
},
tick: function (t, dt) {
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("l_handMeshNode") ) return
const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist")
let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z)
console.log( sum )
if ( sum < .3 ) cubetest.setAttribute("position", AFRAME.utils.coordinates.stringify( wrist.position ) ) // doesn't look good, cube on wrist is moving quite a bit too
// could emit event too
// could check if all joints have close to 0 rotation on ...
// are roughly on the same y-plane of the wrist (facing up or down)
}
})
AFRAME.registerComponent('verticalflatpalmrighthand', {
schema: {
command: {type: 'string'},
},
init: function () {
this.tick = AFRAME.utils.throttleTick(this.tick, 50, this);
window.lastGesture = Date.now() // to initialize
},
tick: function (t, dt) {
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("l_handMeshNode") ) return
const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist")
let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z)
console.log( sum )
//document.querySelector('[isoterminal]').emit("send", [sum + (window.lastGesture+1000>now)]) // +"\n"]) // problematic for special commands
//document.querySelector('[isoterminal]').emit("send", ["\n"]) // kind of horizontal
if ( sum < .3 ) {
// spams the terminal
// could emit event too
let now = Date.now()
if ( now - window.lastGesture > 1000 ){
document.querySelector('[isoterminal]').emit("send", [this.data.command]) // +"\n"]) // problematic for special commands
window.lastGesture = now
AFRAME.scenes[0].emit('gesture', {date:now, type:'vertical flat palm', value:sum})
}
// no event if inhibited due to refractory period
}
if ( sum > 1 ) {
let now = Date.now()
if ( now - window.lastGesture > 1000 ){
document.querySelector('[isoterminal]').emit("send", ["\n"]) // kind of horizontal
window.lastGesture = now
AFRAME.scenes[0].emit('gesture', {date:now, type:'horizontal flat palm', value:sum})
}
// no event if inhibited due to refractory period
}
// could check if all joints have close to 0 rotation on ...
// are roughly on the same y-plane of the wrist (facing up or down)
}
})
AFRAME.registerComponent('gestures', {
init: function () {
this.tick = AFRAME.utils.throttleTick(this.tick, 50, this);
// maybe every 50ms is too often, or not often enough, to check
},
// consider also tock, cf https://aframe.io/docs/1.7.0/core/component.html#before-after
tick: function (t, dt) {
// consider here the order or hierarchy
// optimisation? reconciliation? cascading?
// example of context, e.g. using distance of wrist to a target object, or time, or any other condition
// could be defined else as long as it respect a specific format, like filters/converters/etc
// starting with emiting an event, e.g.
// AFRAME.scenes[0].emit('gesture', {date:now, type:'horizontal flat palm', value:sum})
// or context (unsure if truly has to be separated)
// AFRAME.scenes[0].emit('gesturecontext', {date:now, type:'right wrist in volume of target' })
// detect gesture A
// emit event
// detect gesture B
// emit event
// detect gesture C
// emit event
},
})
AFRAME.registerComponent('gesture-listneres', {
events: {
gesture : function (event) {
console.log( 'gesture captured:', event.detail )
},
gesturecontext : function (event) {
console.log( 'gesture context captured:', event.detail )
}
}
})
AFRAME.registerComponent('positional-context', {
schema: {
target: {type: 'selector'},
attribute: {type: 'string'},
value: {type: 'string'},
threshold: {type: 'float'}
},
init: function () {
this.tick = AFRAME.utils.throttleTick(this.tick, 500, this);
},
tick: function (t, dt) {
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("l_handMeshNode") ) return
const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist").position
// console.log ( this.data.target.object3D.position, this.data.threshold, wrist.distanceTo( this.data.target.object3D.position ) < this.data.threshold )
if ( wrist.distanceTo( this.data.target.object3D.position ) < this.data.threshold ){
// could emit event too
AFRAME.scenes[0].setAttribute(this.data.attribute, "command:"+ this.data.value)
// wrong... this should still delegate instead to a gesture (or gestures) that would THEN conditionally pass the command
AFRAME.scenes[0].emit('gesturecontext', {date:now, type:'right wrist in volume of target' })
// could be used to check for enter/leave based on previous state until now
} else {
AFRAME.scenes[0].removeAttribute(this.data.attribute)
}
}
})
setTimeout( _ => {
// username based to isolate testing
if (username && username == "cubepull") {
AFRAME.scenes[0].setAttribute("cube-pull", "")
}
// multi-users as multi-gestures?
// cascading gestures
console.log ( window.location, window.location.search.includes("xrsh") )
//if ( window.location.search.includes("xrsh") ) {
if ( window.location.host.includes("fabien.benetou.fr") ) {
// URL modified by XRSH
// if (username && username == "xrsh") {
let el = document.createElement("a-sphere")
el.id = "contextsphere"
el.setAttribute("radius", "0.3") // fails on .3 or 0.3
//el.setAttribute("scale", "0.3)
el.setAttribute("wireframe", "true")
el.setAttribute("position", "0 1.4 -1")
AFRAME.scenes[0].appendChild(el)
console.log( 1337 )
let cube = document.createElement("a-box")
cube.setAttribute("target", "") // no effect, probably over written by XRSH
cube.setAttribute("color", "green")
cube.id = "cubetest"
cube.setAttribute("scale", ".1 .1 .1")
cube.setAttribute("position", "0 1.4 -1")
AFRAME.scenes[0].appendChild(cube)
AFRAME.scenes[0].setAttribute("positional-context", "target:#contextsphere; threshold:0.3; attribute:verticalflatpalmrighthand; value: ls -l\\n;")
AFRAME.scenes[0].setAttribute('gesture-listneres','')
// gesture to
// go back in history (then down again)
// to execute
/*
// enter
document.querySelector('[isoterminal]').emit("send", ["\n"])
// arrow keyup
document.querySelector('[isoterminal]').emit("send", ["\x1b[A"])
// arrow keydown
document.querySelector('[isoterminal]').emit("send", ["\x1b[B"])
*/
}
// that can also be visible, e.g. wireframe
if (username && username == "cubepullwithincontext") {
let cube = addCubeWithAnimations()
cube.setAttribute("target", "")
//cube.setAttribute("visible", "false")
el = document.createElement("a-sphere")
el.id = "contextsphere"
el.setAttribute("radius", .3)
el.setAttribute("wireframe", "true")
el.setAttribute("position", "0 1.4 -1")
AFRAME.scenes[0].appendChild(el)
AFRAME.scenes[0].setAttribute("positional-context", "target:#contextsphere; threshold:0.3; attribute:cube-pull;")
}
// that can also be visible, e.g. wireframe
// could do so with example of index finger tip within range of targets, if so make a visual change
// on within cube, off if outside of it
// could change the cube color to show the difference too
// try perf on Quest1
}, 1000 )
// should be a component instead...
setTimeout( _ => {
const myScene = AFRAME.scenes[0].object3D
/*
setInterval( i => {
if (!myScene.getObjectByName("l_handMeshNode") ) return
const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist")
let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z)
console.log( sum )
if ( sum < .3 ) cubetest.setAttribute("position") = wrist.position // doesn't look good, cube on wrist is moving quite a bit too
// could check if all joints have close to 0 rotation on ...
// are roughly on the same y-plane of the wrist (facing up or down)
}, 500 )
*/
/*
gestureThumbEndingAnyContact = setInterval( i => {
if (!myScene.getObjectByName("l_handMeshNode") ) return
// potential shortcuts :
const leftHandJoints = myScene.getObjectByName("l_handMeshNode").parent.children.filter( e => e.type == "Bone")
const rightHandJoints = myScene.getObjectByName("r_handMeshNode").parent.children.filter( e => e.type == "Bone")
const allHandsJoints = leftHandJoints.concat( rightHandJoints )
let posA = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position
let contactPointsToThumbA = leftHandJoints
.concat( rightHandJoints[0].parent.children.filter( e => e.name != 'thumb-tip' ) ) // right hand except thumb tip
.map( e => e.position.distanceTo(posA) ).filter( d => d < .02 ) // threshold of 2cm distance (must be bigger than thumb-tip to previous join position)
// relatively compact description and maybe relatively computively cheap
let pos = myScene.getObjectByName("l_handMeshNode").parent.getObjectByName("thumb-tip").position
let contactPointsToThumb = rightHandJoints
.concat( leftHandJoints[0].parent.children.filter( e => e.name != 'thumb-tip' ) ) // right hand except thumb tip
.map( e => e.position.distanceTo(pos) ).filter( d => d < .02 ) // threshold of 2cm distance (must be bigger than thumb-tip to previous join position)
if (contactPointsToThumb.length+contactPointsToThumbA.length < 1) console.log('no contact'); else console.log('thumb tip in contact with same hand or other hand')
// on contact could also return the join number/names
}, 500 )
*/
/*
testAvegageValue = setInterval( i => {
if (!myScene.getObjectByName("r_handMeshNode") ) return
let rt = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip");
debugValue.x.push( rt.position.x )
let v = debugValue.x
const windowSize = 10 // otherwise too long, e.g 100x500ms gives 5s average
if (v.length > windowSize) {
values = v.slice(-windowSize)
let avg = ( values.reduce( (acc,c) => acc+c )/windowSize) .toFixed(3)
console.log( avg )
}
}, 50 )
*/
/*
showContactPoints = setInterval( i => {
if (!myScene.getObjectByName("r_handMeshNode") ) return
let targetJoints = []
tips.map( t => targetJoints.push( myScene.getObjectByName("r_handMeshNode").parent.getObjectByName(t) ) )
tips.map( t => targetJoints.push( myScene.getObjectByName("l_handMeshNode").parent.getObjectByName(t) ) )
// tips only
let contacts = proximityBetweenJointsCheck(targetJoints)
if (contacts.length) {
console.log( "contacts:", contacts )
// {dist: dist.toFixed(3), a:j[0].name, ah: j[0].parent.parent.el.id, b:j[1].name, bh: j[1].parent.parent.el.id }
contacts.map( c => {
// show value or even just a temporary object there
let a = document.getElementById(c.ah).object3D.getObjectByName(c.a)
let b = document.getElementById(c.bh).object3D.getObjectByName(c.b)
const geometry = new THREE.BoxGeometry( .01, .01, .01 )
const material = new THREE.MeshBasicMaterial( { color: 0x00ff00 } )
const cube = new THREE.Mesh( geometry, material )
a.add( cube )
})
}
})
*/
/*
showGestureDistanceDebugJoints = setInterval( i => {
if (!myScene.getObjectByName("r_handMeshNode") ) return
let targetJoints = []
tips.map( t => targetJoints.push( myScene.getObjectByName("r_handMeshNode").parent.getObjectByName(t) ) )
tips.map( t => targetJoints.push( myScene.getObjectByName("l_handMeshNode").parent.getObjectByName(t) ) )
// console.log( targetJoints ) looks fine
//console.log( "contacts:", proximityBetweenJointsCheck(targetJoints)
// tips only
let targetJointsFull = []
allJointsNames.map( t => targetJointsFull.push( myScene.getObjectByName("r_handMeshNode").parent.getObjectByName(t) ) )
allJointsNames.map( t => targetJointsFull.push( myScene.getObjectByName("l_handMeshNode").parent.getObjectByName(t) ) )
let contacts = proximityBetweenJointsCheck(targetJointsFull)
if (contacts.length) console.log( "contacts:", contacts )
})
*/
/*
showGestureDistanceDebug = setInterval( i => {
if (!myScene.getObjectByName("r_handMeshNode") ) return
let rt = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position
let lt = myScene.getObjectByName("l_handMeshNode").parent.getObjectByName("thumb-tip").position
if ( rt.distanceTo(lt) < .1 )
console.log( 'lt close to rt')
else
console.log( rt.distanceTo(lt) )
})
*/
/*
showGestureDebug = setInterval( i => {
if (!myScene.getObjectByName("r_handMeshNode") ) return
let rt = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip");
//console.log( shortVec3( rt.position ), shortVec3( rt.rotation ) )
// could do for the 2x25 values... but then becomes unreadible, hence why showing sparklines could help
// can be done on HUD
if (!debugValue.x){
debugValue.x = []
debugValue.y = []
debugValue.z = []
debugValue.a = []
debugValue.b = []
debugValue.c = []
}
debugValue.x.push( rt.position.x )
debugValue.y.push( rt.position.y )
debugValue.z.push( rt.position.z )
debugValue.a.push( rt.rotation.x )
debugValue.b.push( rt.rotation.y )
debugValue.c.push( rt.rotation.z )
let el = document.getElementById("debuggraph")
if (!el) addDebbugGraph()
drawPoints( debugValue )
}, 50 )
*/
}, 1000)
// waiting for the scene to be loaded, could be component proper too...

167
src/gestures.html Normal file
View file

@ -0,0 +1,167 @@
<html>
<head>
<script>
// for filters/keymap.js setting up
let sequentialFilters = [] // the ability to parse files sequentially, even though here only for one filter
const urlParams = new URLSearchParams(window.location.search); // in URL parameters
let filesWithMetadata = {} // files metadata
</script>
<script src="https://companion.benetou.fr/filters/keymap.js"></script>
<script src="https://aframe.io/releases/1.7.1/aframe.min.js"></script>
<script src="https://cdn.jsdelivr.net/gh/kylebakerio/a-console@1.0.2/a-console.js"></script>
<script src="https://unpkg.com/aframe-troika-text/dist/aframe-troika-text.min.js"></script>
<script src="https://xrsh.isvery.ninja/xrsh.js"></script>
<script src="jxr-core.js"></script>
<script src="drumkbd.js"></script>
<script src="uglyph.js"></script>
<script src="uglyph_tests.js"></script>
</head>
<body>
<a-scene>
<a-sky color=lightblue></a-sky>
<a-entity isoterminal position="0 1.6 -0.3"></a-entity>
<a-entity id="rig">
<!-- use to get hand tracking for keyboard input -->
<a-entity id="player" hud camera look-controls wasd-controls position="0 1.6 0"></a-entity>
<a-entity id="rightHand" hand-tracking-controls="hand: right;"></a-entity>
<a-entity id="leftHand" hand-tracking-controls="hand: left;"></a-entity>
</a-entity>
<a-console skip-intro="true" rotation="0 -70 0" height=.3 position=".7 1.5 0"></a-console>
</a-scene>
<script>
document.querySelector('a-scene').addEventListener('isoterminal_init', function(e){
// (OPTIONAL) override/extend bootmenu-array with your own REPL(s)
// see com/isoterminal/feat/boot.REPL.*.js for examples
ISOTerminal.prototype.boot.menu.push({
key: "f",
title: (opts) => `other awesome NLnet FOSS projects ❤️`,
init: function( mainmenu ){
window.open('https://nlnet.nl/project/current.html','_blank')
// return to main menu
mainmenu()
},
keyHandler: function(ch){
this.send("\n\ryou typed:"+ch) // write to term
}
})
})
// video demo https://video.benetou.fr/w/dWxnUspg6oAUgYLo6QuoUU
setTimeout( _ => {
// component 'bind-element-to-finger' from uglyph.js required for keyboard interactions
// usable as e.g hand: 'r_handMeshNode', finger: 'index-finger-tip', target : 'selector'
/*
window.keyboardTarget = typinghud
let kbd = addDrumKeyboard()
kbd.addEventListener( "keypressed", e => {
document.querySelector("[isoterminal]").components.isoterminal.term.send(e.detail.key)
// consider executing the content on ENTER
// document.querySelector("[isoterminal]").components.isoterminal.term.exec("ls")
// raw keypress, this getting SHFT, ENT, etc
})
*/
// testing from uglyph.js
// addDebbugGraph() // needed by showGestureDebug()
setInterval( _ => {
//showGestureDebug()
//pullCubeToWrist() // probably needs a cube to pull...
//gestureThumbEndingAnyContact()
//testAvegageValue()
//showContactPoints()
//showGestureDistanceDebug()
//showGestureDistanceDebugJoints()
//secondaryPinchSingleHanded()
}, 50 )
// console.log('available detectors:', detectors)
// to be wrapped in setInterval() or AFrame component with tick method
// (prints as empty in <a-console> but print function names properly in Firefox console)
/* events to listen to from detector
'secondary-singled-handed-pinch-not'
'secondary-singled-handed-pinch'
'secondary-singled-handed-pinch-middle-finger'
AFRAME.scenes[0].addEventListener('secondary-singled-handed-pinch', evt => {
console.log(evt.detail) // here getting the hand, e.g. right
document.querySelector("[isoterminal]").components.isoterminal.term.send(evt.detail.hand[0])
// symbolic demonstration, get "r" here appended to terminal input
} )
*/
//window.test = function( event ){ console.log('custom callback showcase:', event.detail) }
//AFRAME.scenes[0].setAttribute('template-gesture-detector', 'callback:test;') // no customization
let el = document.createElement("a-box")
el.id = "demotarget"
el.setAttribute("scale", ".1 .1 .1")
el.setAttribute("color", "darkgreen")
el.setAttribute("position", "0 1.6 -1")
AFRAME.scenes[0].appendChild(el)
AFRAME.scenes[0].setAttribute('template-gesture-detector', 'target:#demotarget;') // no customization
/*
AFRAME component 'template-gesture-detector' :
threshold: {type: 'number', default: .3}, // arbitrary unit, e.g distance in m, sum of angles in degrees, etc
frequency: {type: 'number', default: 50}, // in ms
target: {type: 'selector'}, // common way to apply the transformation to the result
callback: {type: 'string', default: ''},
*/
/*
AFRAME.scenes[0].setAttribute('verticalflatpalmrighthand', '')
// AFRAME.scenes[0].setAttribute('gesture-listeners', '') // console.log is a VERY big performance hit
AFRAME.registerComponent('verticalflatpalmrighthand-listener', {
events: {
gesture : function (event) {
if (event.detail.type == 'vertical flat palm') // only type here, just to showcase filtering
AFRAME.scenes[0].querySelector("[isoterminal]").object3D.children[0].material.color
// AFRAME.scenes[0].querySelector("a-sky").object3D.children[0].material.color
.setRGB(.5,.5,event.detail.value/3)
},
}
})
AFRAME.scenes[0].setAttribute('verticalflatpalmrighthand-listener', '')
*/
// testCubePullWithinContext() // works only inside sphere
// testVerticalFlatRightHand() // same principle, different gesture detector and effect
// calls gesture-listeners itself
/* components to add
'cube-pull' // worked from anywhere
'gestures'
'positional-context'
*/
// component wrapping suggestion
/*
const componentNameExample = "something-unique"
AFRAME.registerComponent(componentNameExample, {
init: function () { this.tick = AFRAME.utils.throttleTick(this.tick, 50, this) },
tick: function (time, timeDelta) { gestureThumbEndingAnyContact() }
// output visible in the console, hence needing e.g <a-console>
})
AFRAME.scenes[0].setAttribute(componentNameExample, '')
*/
}, 3000 )
// see also microgestures https://registry.khronos.org/OpenXR/specs/1.1/html/xrspec.html#XR_META_hand_tracking_microgestures
</script>
</body>
</html>

View file

@ -1,480 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>xrsh v0.143</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
<link rel="stylesheet" href="./index.css"></link>
<script src="./assets/aframe.min.js"></script>
<script src="com/require.js"></script>
<script src="com/isoterminal.js"></script>
<!-- todo fix lazy initialisation of require.js -->
<script src="com/launcher.js"></script>
<script src="com/helloworld-window.js"></script>
<!--
conflict but doesn't prevent the rest from executing
<script src="com/pressable.js"></script>
-->
<script src="https://cdn.jsdelivr.net/npm/aframe-troika-text/dist/aframe-troika-text.min.js"></script>
<script src="https://cdn.jsdelivr.net/gh/kylebakerio/a-console@1.0.2/a-console.js"></script>
<script src="https://fabien.benetou.fr/pub/home/future_of_text_demo/engine/jxr-core_branch_teleport_alt_rot.js?version=cachebusing123455"></script>
<script src="https://fabien.benetou.fr/pub/home/future_of_text_demo/engine/jxr-postitnote.js"></script>
<script src="gesture-exploration.js"></script>
<script>
/*
gesture manager :
eventRecipient (if not specified document? scene?)
eventName
functionName(parameters?) // event details expected?
examples
on pinch from any hand console.log( event details)
*/
const urlParams = new URLSearchParams(window.location.search);
const username = urlParams.get('username');
localStorage.setItem("restorestate","true")
AFRAME.registerComponent('useraddednote', {
events: {
useraddednote: function (e) {
let noteEl = e.detail.element
if ( noteEl.getAttribute("value").startsWith("jxr ") ) return
noteEl.classList.add("manuscriptnote")
// dirty mix of threejs and AFrame...
noteEl.object3D.parent = manuscript.object3D;
//noteEl.setAttribute("position", manuscript.children[0].getAttribute("position") )
//noteEl.setAttribute("rotation", manuscript.children[0].getAttribute("rotation") )
setTimeout( _ => noteEl.object3D.position.set( -.4, .4 + - document.querySelectorAll(".manuscriptnote").length/10, .51), 100 )
// messes up direct picking after, so could do an interaction filter on pick for this class
// relatively complex to keep track of but should work
// could until then prevent picking, e.g. removing the target
},
}
})
function sceneIdsToElementForRings(){
// does not actually traverse the scene, only gets the top level entities
Array.from( document.querySelector("a-scene").childNodes ).filter( el => el.id && !el.id.startsWith("note_") ).reverse().map( (el,i) => {
let noteEl = addNewNote("#"+el.id, "1 "+(1+i/50)+" -1")
noteEl.setAttribute("onpicked","startRingCheck()")
noteEl.setAttribute("onreleased","endRingCheck()")
})
}
let ringCheckInterval = null
let ringAdded = false
function startRingCheck(){
// document.querySelector("a-console").setAttribute("visible", true)
let pos = new THREE.Vector3()
let el = selectedElements.at(-1)?.element
if (!el) return
let historyPositions = []
document.querySelector("a-console").setAttribute("visible", "true")
ringCheckInterval = setInterval( el => {
el = selectedElements.at(-1)?.element
el.object3D.getWorldPosition( pos )
/*
// trailing line
if ( AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode") ){
historyPositions.push( AFRAME.utils.coordinates.stringify( AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position ) )
if (historyPositions.length > 10){
let path = historyPositions.slice(-10).join(", ")
console.log( path )
rings.querySelector("a-tube").setAttribute("path", path)
console.log( rings.querySelector("a-tube") )
}
}
*/
Array.from( active_rings.children ).map( r => {
r.setAttribute("animation", "property: rotation.z; from: 0; to: 360; dur: 1000; startEvents:startAnimation;")
let pos_ring = new THREE.Vector3()
r.object3D.getWorldPosition( pos_ring )
let d = pos.distanceTo( pos_ring )
if (d<.2) {
// visual feedback on proximity
r.setAttribute("wireframe", "true")
} else {
r.setAttribute("wireframe", "false")
}
if (d<.1) {
// should append as clone of r to rings_stack
// ding sound
// audio.play()
// restart every time
r.emit('startAnimation', null, false)
let elValue = el.getAttribute("value")
// to clean up otherwise always expect a value, e.g. on gltf-model
// ---------------------------------------- resolving -----------------------------------------------------
let resolvedElements = []
if ( elValue && elValue.length > 0 && elValue.startsWith(".") || elValue.startsWith("#") || elValue.startsWith("a-") ){
if ( elValue.startsWith(".") ){
resolvedElements = Array.from( AFRAME.scenes[0].querySelectorAll(elValue) )
} else {
el = document.querySelector(elValue)
// assuming "a-..." is unique, e.g. a-sky, which isn't true for others, e.g. a-troika-text
}
}
let value = r.querySelector("a-troika-text").getAttribute("value")
// should instead "just" apply the jxr, done here just for testing
// probably need some refractory period too, otherwise stacking transformations that are probably not required
// ---------------------------------------- this should be based on value itself, i.e. another eval() -----------------------------------------------------
// see applyFunctionToSelection() ... which does not exist
// can be replaced by something more generic, there are already jxr shortcuts e.g. sa attributeName attributeValue
// could be done as filters are done, i.e. filters.map( f => f(element) )
resolvedElements.map( rEl => {
if (value.includes("scaleUp") ) rEl.setAttribute("scale", ".01 .02 .02")
if (value.includes("White") ) rEl.setAttribute("color", "white")
if (value.includes("Blue") ) rEl.setAttribute("color", "blue")
if (value.includes("Red") ) rEl.setAttribute("color", "red")
})
// gets busy very quickly, surely because of the lack of refractory period TODO fix
if (false && value.length>1) {
console.log('should clone ', r, ' on stack', rings_stack )
let clonedRing = r.cloneNode(true)
rings_stack.appendChild( clonedRing )
}
if (value.includes("scaleUp") ) el.setAttribute("scale", ".02 .02 .02")
if (value.includes("White") ) el.setAttribute("color", "white")
if (value.includes("Blue") ) el.setAttribute("color", "blue")
if (value.includes("Red") ) el.setAttribute("color", "red")
// assume we are ready to send command (which is often NOT true as it takes quite some time to get the ISO running, even with save)
let temporaryBufferForPipping = ''
// TODO need properly escaping/encapsulating
if (value.includes('isoterminal("ls")') ) document.querySelector('[isoterminal]').emit("exec", ["echo 'hello world'", (str,buf) => { console.log(str); temporaryBufferForPipping = str } ])
if (value.includes('isoterminal(" | wc -l")') ) document.querySelector('[isoterminal]').emit("exec", [ "echo " + temporaryBufferForPipping + " | wc", (str,buf) => console.log(str, " (temporaryBufferForPipping:"+ temporaryBufferForPipping + ")") ])
// temporaryBufferForPipping is empty
// could be tried outside of XR
// problem too, as here we're getting wc 1 0 1 rather than 1 2 12
//if (value.includes('isoterminal(" | wc -l")') ) document.querySelector('[isoterminal]').emit("exec", [ "echo " + temporaryBufferForPipping + " | wc -l", (str,buf) => console.log(str) ])
// newline problem
// could include resolving too, e.g .collidable doesn't get appended as-is but rather its resulting elements do
// could be better to do it later on, at the transformation level, going through ring
if (value.includes("addToRingSelection") ) {
// try to get the selection value and append to it, if none make new one
const selectionNodeId = "selectionnote"
let selectionNoteEl = document.getElementById( selectionNodeId )
if (!selectionNoteEl){
let newPos = r.getAttribute("position").clone()
newPos.y += .3
selectionNoteEl = addNewNote( elValue, AFRAME.utils.coordinates.stringify( newPos ), "0.1 0.1 0.1", selectionNodeId) // text only, not 3D model, for this would try to get its ID instead
selectionNoteEl.setAttribute("onpicked", "startRingCheck()")
selectionNoteEl.setAttribute("onreleased", "endRingCheck()")
} else {
selectionNoteEl.setAttribute("value", selectionNoteEl.getAttribute("value") + "\n" + elValue )
}
}
if (value.length==1) el.setAttribute("value", el.getAttribute("value")+value)
// character append
// could then make a keyboard this way...
// but 26 characters make for very large movements
// assuming a meta-ring but the layout itself can be totally different
// can try much smaller rings
// also need a refractory period
// add another ring to
// create new word (how?)
// split existing word on " "
// ---------------------------------------- -----------------------------------------------------
// should then
// clone ring then append it to rings_stack (for follow up macro)
// this should be permanent, i.e. JSON WebDAV save, and movable, i.e add target attribute (done)
// show/unhide/unfold the "following" rings based on context
// special rings ideas
// key aspect of UI/UX
if (!ringAdded){
// here we are considering a sequential "positive" flow
// but could also be negative, a la ! or conditional e.g. && or ||
// so rings could have a type or return value equivalent
ringAdded = true
// need refractory period or maximum amount of times added
let newRing = addRing("jxr console.log('Red')", r.getAttribute("position") )
newRing.setAttribute("color", "red")
setTimeout( _ => newRing.object3D.position.z += .3, 100 )
// offset from current ring position
// this itself should enable the creation of a 4th ring, etc
}
}
})
}, 100)
}
function addKeyboardRings(){
// addRing(code, position)
"abcdefghijklmnopqrstuvwxyz ,.{}()[]".split('').reverse().map( (c,i) => {
let x = (i%3)/10 +.5
let y = 1+(i/3)/10
addRing(c, x+" "+y+ " -.5", .03, .005, .005)
} )
// could probably be done once they repositioned instead via a well known parent with id (or unique selector)
}
function addRing(code, position="0 1.5 -.7", radius=.1, radiusTubular=.01, codeVerticalOffset=.15){
let el = document.createElement("a-torus")
el.setAttribute("position", position)
el.setAttribute("segments-radial", "4")
el.setAttribute("segments-tubular", "12")
el.setAttribute("opacity", ".3")
el.setAttribute("radius", radius)
el.setAttribute("radius-tubular", radiusTubular)
let elCode = document.createElement("a-troika-text")
// consider binding an existing jxr command to an existing ring to have a similar behavior
// e.g. drop jxr on ring, bind them so that next time an element goes through, it's applied via binded ring
elCode.setAttribute("anchor", "left")
elCode.setAttribute("target", "")
elCode.setAttribute("value", code)
elCode.setAttribute("position", ".0 "+codeVerticalOffset+" .0")
elCode.setAttribute("scale", "0.1 0.1 0.1")
el.appendChild( elCode )
active_rings.appendChild( el )
return el
}
function endRingCheck(){
clearInterval(ringCheckInterval)
}
function applyNextFilterInteraction (){}
let sequentialFiltersInteractionOnPicked = []
let sequentialFiltersInteractionOnReleased = []
let currentFilterOnPicked = null
let currentFilterOnReleased = null
</script>
</head>
<body style="background:linear-gradient( 45deg, #324, #8c74ff)">
<a-scene selfcontainer
light="defaultLightsEnabled: true"
cursor="rayOrigin: mouse"
renderer="colorManagement: false; stencil: true; antialias:true; highRefreshRate:true; foveationLevel: 0.5;"
xr-mode-ui="XRMode: xr"
obb-collider="showColliders:false"
raycaster="objects: [html]; interval:100;"
>
<a-sphere id=contextsphere wireframe=true position="0 1.4 -1" radius=.3></a-sphere>
<a-console position="-1 1.3 0" rotation="-45 90 0" font-size="34" height="1" skip-intro="true"></a-console>
<a-entity isoterminal="iso: https://forgejo.isvery.ninja/assets/xrsh-buildroot/main/xrsh.iso; minimized:true; overlayfs: /package.overlayfs.zip; bootMenu: 1; bootMenuURL: 1" position="0 1.6 -0.3">
<a-entity launcher pressable position="0 -0.315 0"></a-entity>
</a-entity>
<a-sphere position="1 1 -2"></a-sphere>
<a-entity helloworld-window pressable position="0 1.6 0"></a-entity>
<a-entity id="rig">
<a-entity id="player">
<a-entity camera="fov:90" position="0 1.6 0" id="camera" wasd-controls gaze-touch-to-click></a-entity>
<a-entity id="left-hand" hand-tracking-grab-controls="hand: left; modelColor: #EEEEEE; hoverEnabled:true"
laser-controls="hand: left"
raycaster="far:0.04"
blink-controls="cameraRig:#player; teleportOrigin: #camera; collisionEntities: #floor"
pinchsecondary
>
</a-entity>
<a-entity id="right-hand" hand-tracking-grab-controls="hand: right; modelColor: #EEEEEE; hoverEnabled:true"
laser-controls="hand: right"
raycaster="far:0.04"
pinchprimary
blink-controls="cameraRig:#player; teleportOrigin: #camera; collisionEntities: #floor"
____pinch-to-teleport="rig: #player"></a-entity>
</a-entity>
</a-entity>
<a-sphere segments-width=12 segments-height=12 pressable="" start-on-press="" id="box" radius="0.033" color="gray"></a-sphere>
<a-box pressable="" start-on-press-other="" id="otherbox" scale=".05 .05 .05" opacity=.3 wireframe=false color="white"></a-box>
<a-entity id=rings>
<!-- consider applyNextFilterInteraction() -->
<!-- sequence of rings, namely no "absolute" list, in addition to apply a function to the content currently selected, it opens up the "next" rings and stacks itself to the applied rings -->
<a-troika-text anchor=left target value= "jxr addKeyboardRings()" position="0 1.10 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text anchor=left target value= 'jxr Array.from( active_rings.querySelectorAll("a-torus") ).map( el => { el.setAttribute("radius", ".01"); el.setAttribute("radius-tubular", ".001") })' position="0.4 1.20 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value= "jxr sceneIdsToElementForRings()" position="0 1.20 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='example text' position="0 1.30 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='a-sky' position="0 1.40 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='.collidable' position="0 1.60 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position="0 1.50 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-entity id=active_rings>
<a-torus color="green" position="0.4 1.4 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr isoterminal("ls")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="green" position="0.4 1.4 -.8" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr isoterminal(" | wc -l")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="blue" position="0 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="white" position="0 1 -.3" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToWhite")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="white" position=".5 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr scaleUp()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="white" position="-.5 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr addRingFromCode()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="white" position="-.5 1.5 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr addToRingSelection()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
</a-entity>
<!-- see history of transformations via voice, and cancelling it, older PoC -->
<a-entity id=rings_stack target position="-1 1 -1.5">
<a-troika-text anchor=left target value='previous transformations' position="-.1 .3 .1" scale="0.1 0.1 0.1"></a-troika-text>
<a-torus wireframe=true color="#43A367" position="0 0 -.1" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text curve-radius=1 rotation="90 0 0" anchor=left target value='jxr applyFunctionToSelection("changeColorToRed")' position=".0 .1 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus wireframe=true color="#43A367" position="0 0 0" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
</a-entity>
</a-entity>
</a-scene>
</body>
<script>
const term=document.querySelector('[isoterminal]');
term.emit("exec", ["ls -l", (str,buf) => console.log(str) ])
// HOWDY DEVELOPER!
// for a REPL boilerplate see this codepen instead: https://codepen.io/coderofsalvation/pen/wBwRpew
document.querySelector('a-scene').addEventListener('isoterminal_init', function(e){
const term=document.querySelector('[isoterminal]');
term.addEventListener('ready', e => {
term.emit("exec", ["ls -l", (str,buf) => console.log(str) ])
})
// insert rings here instead
// example custom REPL (https://xrsh.isvery.ninja/#Custom%20REPLs)
const myREPL = {
key: "f",
prompt: "\r\nmyrepl> ",
title: (opts) => `other awesome NLnet FOSS projects ❤️`,
init: function( mainmenu ){
window.open('https://nlnet.nl/project/current.html','_blank')
this.send("\n\rreturning to mainmenu")
mainmenu()
},
keyHandler: function(ch){
this.send(ch) // echo keys to user
},
cmdHandler: function(cmd){
this.send("\r\n"+cmd) // only works after commenting mainmenu()\n")
this.send(myREPL.prompt)
}
}
ISOTerminal.prototype.boot.menu.push(myREPL)
})
setTimeout( _ => {
let jointTestEl1 = addNewNote( "jxr document.querySelector('a-sky').setAttribute('color','purple')", "0 .03 -.05")
jointTestEl1.setAttribute("scale", ".05 .05 .05")
jointTestEl1.setAttribute("rotation", "0 -90 0")
jointTestEl1.setAttribute("target", "")
jointTestEl1.id = "jointtest1"
let jointTestEl2 = addNewNote( "jxr document.querySelector('a-sky').setAttribute('color','gray')", "0 .03 -.05")
jointTestEl2.setAttribute("scale", ".05 .05 .05")
jointTestEl2.setAttribute("position", "0 .03 -.05")
jointTestEl2.setAttribute("rotation", "0 -90 0")
jointTestEl2.id = "jointtest2"
AFRAME.scenes[0].setAttribute("bind-element-to-finger__test1", { hand: 'r_handMeshNode', finger: 'index-finger-tip', target: '#jointtest1' })
AFRAME.scenes[0].setAttribute("bind-element-to-finger__test2", { hand: 'r_handMeshNode', finger: 'thumb-tip', target: '#jointtest2' } )
}, 1000)
AFRAME.registerComponent('bind-element-to-finger', {
multiple: true,
schema: {
hand: {type: 'string', default: 'r_handMeshNode'},
finger: {type: 'string', default: 'index-finger-tip'},
target : {type: 'selector'},
},
init: function () {
},
tick: function (time, timeDelta) {
const joint = AFRAME.scenes[0].object3D.getObjectByName(this.data.hand)?.parent.getObjectByName(this.data.finger)
if ( joint && this.data.target.object3D.parent == AFRAME.scenes[0].object3D ) this.data.target.object3D.parent = joint
// unfortunately reparenting for now breaks properly picking...
// could check in core for whom the parent is and reparent only on release
// or potentially not at all, maybe the user expects to be able to pick the instruction away from the hand
// would also then need an explicit mechanism to bind back
// a la wrist shortcut
// warning that some parenting is done "right" namely by removing the offset then become an A-Frame child with that offset
// allowing thus the expected behavior in that context
// consequently might have to distinguish the 2 situations
}
})
/* // does not work reliably, seems to be just the 1st keypress
var keyEvent = document.createEvent('KeyboardEvent');
keyEvent.initKeyboardEvent('keydown', true, false, null, 0, false, 0, false, 77, 0);
window.dispatchEvent(keyEvent);
*/
</script>
</html>

View file

@ -1,482 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>xrsh v0.143</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
<link rel="stylesheet" href="./index.css"></link>
<script src="./assets/aframe.min.js"></script>
<script src="com/require.js"></script>
<script src="com/isoterminal.js"></script>
<!-- todo fix lazy initialisation of require.js -->
<script src="com/launcher.js"></script>
<script src="com/helloworld-window.js"></script>
<!--
conflict but doesn't prevent the rest from executing
<script src="com/pressable.js"></script>
-->
<script src="https://cdn.jsdelivr.net/npm/aframe-troika-text/dist/aframe-troika-text.min.js"></script>
<script src="https://cdn.jsdelivr.net/gh/kylebakerio/a-console@1.0.2/a-console.js"></script>
<script src="https://fabien.benetou.fr/pub/home/future_of_text_demo/engine/jxr-core_branch_teleport_alt_rot.js?version=cachebusing123455"></script>
<script src="https://fabien.benetou.fr/pub/home/future_of_text_demo/engine/jxr-postitnote.js"></script>
<script src="https://companion.benetou.fr/gesture-exploration.js?version=cachebusting1234"></script>
<script>
/*
gesture manager :
eventRecipient (if not specified document? scene?)
eventName
functionName(parameters?) // event details expected?
examples
on pinch from any hand console.log( event details)
*/
const urlParams = new URLSearchParams(window.location.search);
const username = urlParams.get('username');
localStorage.setItem("restorestate","true")
AFRAME.registerComponent('useraddednote', {
events: {
useraddednote: function (e) {
let noteEl = e.detail.element
if ( noteEl.getAttribute("value").startsWith("jxr ") ) return
noteEl.classList.add("manuscriptnote")
// dirty mix of threejs and AFrame...
noteEl.object3D.parent = manuscript.object3D;
//noteEl.setAttribute("position", manuscript.children[0].getAttribute("position") )
//noteEl.setAttribute("rotation", manuscript.children[0].getAttribute("rotation") )
setTimeout( _ => noteEl.object3D.position.set( -.4, .4 + - document.querySelectorAll(".manuscriptnote").length/10, .51), 100 )
// messes up direct picking after, so could do an interaction filter on pick for this class
// relatively complex to keep track of but should work
// could until then prevent picking, e.g. removing the target
},
}
})
function sceneIdsToElementForRings(){
// does not actually traverse the scene, only gets the top level entities
Array.from( document.querySelector("a-scene").childNodes ).filter( el => el.id && !el.id.startsWith("note_") ).reverse().map( (el,i) => {
let noteEl = addNewNote("#"+el.id, "1 "+(1+i/50)+" -1")
noteEl.setAttribute("onpicked","startRingCheck()")
noteEl.setAttribute("onreleased","endRingCheck()")
})
}
let ringCheckInterval = null
let ringAdded = false
function startRingCheck(){
// document.querySelector("a-console").setAttribute("visible", true)
let pos = new THREE.Vector3()
let el = selectedElements.at(-1)?.element
if (!el) return
let historyPositions = []
document.querySelector("a-console").setAttribute("visible", "true")
ringCheckInterval = setInterval( el => {
el = selectedElements.at(-1)?.element
el.object3D.getWorldPosition( pos )
/*
// trailing line
if ( AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode") ){
historyPositions.push( AFRAME.utils.coordinates.stringify( AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position ) )
if (historyPositions.length > 10){
let path = historyPositions.slice(-10).join(", ")
console.log( path )
rings.querySelector("a-tube").setAttribute("path", path)
console.log( rings.querySelector("a-tube") )
}
}
*/
Array.from( active_rings.children ).map( r => {
r.setAttribute("animation", "property: rotation.z; from: 0; to: 360; dur: 1000; startEvents:startAnimation;")
let pos_ring = new THREE.Vector3()
r.object3D.getWorldPosition( pos_ring )
let d = pos.distanceTo( pos_ring )
if (d<.2) {
// visual feedback on proximity
r.setAttribute("wireframe", "true")
} else {
r.setAttribute("wireframe", "false")
}
if (d<.1) {
// should append as clone of r to rings_stack
// ding sound
// audio.play()
// restart every time
r.emit('startAnimation', null, false)
let elValue = el.getAttribute("value")
// to clean up otherwise always expect a value, e.g. on gltf-model
// ---------------------------------------- resolving -----------------------------------------------------
let resolvedElements = []
if ( elValue && elValue.length > 0 && elValue.startsWith(".") || elValue.startsWith("#") || elValue.startsWith("a-") ){
if ( elValue.startsWith(".") ){
resolvedElements = Array.from( AFRAME.scenes[0].querySelectorAll(elValue) )
} else {
el = document.querySelector(elValue)
// assuming "a-..." is unique, e.g. a-sky, which isn't true for others, e.g. a-troika-text
}
}
let value = r.querySelector("a-troika-text").getAttribute("value")
// should instead "just" apply the jxr, done here just for testing
// probably need some refractory period too, otherwise stacking transformations that are probably not required
// ---------------------------------------- this should be based on value itself, i.e. another eval() -----------------------------------------------------
// see applyFunctionToSelection() ... which does not exist
// can be replaced by something more generic, there are already jxr shortcuts e.g. sa attributeName attributeValue
// could be done as filters are done, i.e. filters.map( f => f(element) )
resolvedElements.map( rEl => {
if (value.includes("scaleUp") ) rEl.setAttribute("scale", ".01 .02 .02")
if (value.includes("White") ) rEl.setAttribute("color", "white")
if (value.includes("Blue") ) rEl.setAttribute("color", "blue")
if (value.includes("Red") ) rEl.setAttribute("color", "red")
})
// gets busy very quickly, surely because of the lack of refractory period TODO fix
if (false && value.length>1) {
console.log('should clone ', r, ' on stack', rings_stack )
let clonedRing = r.cloneNode(true)
rings_stack.appendChild( clonedRing )
}
if (value.includes("scaleUp") ) el.setAttribute("scale", ".02 .02 .02")
if (value.includes("White") ) el.setAttribute("color", "white")
if (value.includes("Blue") ) el.setAttribute("color", "blue")
if (value.includes("Red") ) el.setAttribute("color", "red")
// assume we are ready to send command (which is often NOT true as it takes quite some time to get the ISO running, even with save)
let temporaryBufferForPipping = ''
// TODO need properly escaping/encapsulating
if (value.includes('isoterminal("ls")') ) document.querySelector('[isoterminal]').emit("exec", ["echo 'hello world'", (str,buf) => { console.log(str); temporaryBufferForPipping = str } ])
if (value.includes('isoterminal(" | wc -l")') ) document.querySelector('[isoterminal]').emit("exec", [ "echo " + temporaryBufferForPipping + " | wc", (str,buf) => console.log(str, " (temporaryBufferForPipping:"+ temporaryBufferForPipping + ")") ])
// temporaryBufferForPipping is empty
// could be tried outside of XR
// problem too, as here we're getting wc 1 0 1 rather than 1 2 12
//if (value.includes('isoterminal(" | wc -l")') ) document.querySelector('[isoterminal]').emit("exec", [ "echo " + temporaryBufferForPipping + " | wc -l", (str,buf) => console.log(str) ])
// newline problem
// could include resolving too, e.g .collidable doesn't get appended as-is but rather its resulting elements do
// could be better to do it later on, at the transformation level, going through ring
if (value.includes("addToRingSelection") ) {
// try to get the selection value and append to it, if none make new one
const selectionNodeId = "selectionnote"
let selectionNoteEl = document.getElementById( selectionNodeId )
if (!selectionNoteEl){
let newPos = r.getAttribute("position").clone()
newPos.y += .3
selectionNoteEl = addNewNote( elValue, AFRAME.utils.coordinates.stringify( newPos ), "0.1 0.1 0.1", selectionNodeId) // text only, not 3D model, for this would try to get its ID instead
selectionNoteEl.setAttribute("onpicked", "startRingCheck()")
selectionNoteEl.setAttribute("onreleased", "endRingCheck()")
} else {
selectionNoteEl.setAttribute("value", selectionNoteEl.getAttribute("value") + "\n" + elValue )
}
}
if (value.length==1) el.setAttribute("value", el.getAttribute("value")+value)
// character append
// could then make a keyboard this way...
// but 26 characters make for very large movements
// assuming a meta-ring but the layout itself can be totally different
// can try much smaller rings
// also need a refractory period
// add another ring to
// create new word (how?)
// split existing word on " "
// ---------------------------------------- -----------------------------------------------------
// should then
// clone ring then append it to rings_stack (for follow up macro)
// this should be permanent, i.e. JSON WebDAV save, and movable, i.e add target attribute (done)
// show/unhide/unfold the "following" rings based on context
// special rings ideas
// key aspect of UI/UX
if (!ringAdded){
// here we are considering a sequential "positive" flow
// but could also be negative, a la ! or conditional e.g. && or ||
// so rings could have a type or return value equivalent
ringAdded = true
// need refractory period or maximum amount of times added
let newRing = addRing("jxr console.log('Red')", r.getAttribute("position") )
newRing.setAttribute("color", "red")
setTimeout( _ => newRing.object3D.position.z += .3, 100 )
// offset from current ring position
// this itself should enable the creation of a 4th ring, etc
}
}
})
}, 100)
}
function addKeyboardRings(){
// addRing(code, position)
"abcdefghijklmnopqrstuvwxyz ,.{}()[]".split('').reverse().map( (c,i) => {
let x = (i%3)/10 +.5
let y = 1+(i/3)/10
addRing(c, x+" "+y+ " -.5", .03, .005, .005)
} )
// could probably be done once they repositioned instead via a well known parent with id (or unique selector)
}
function addRing(code, position="0 1.5 -.7", radius=.1, radiusTubular=.01, codeVerticalOffset=.15){
let el = document.createElement("a-torus")
el.setAttribute("position", position)
el.setAttribute("segments-radial", "4")
el.setAttribute("segments-tubular", "12")
el.setAttribute("opacity", ".3")
el.setAttribute("radius", radius)
el.setAttribute("radius-tubular", radiusTubular)
let elCode = document.createElement("a-troika-text")
// consider binding an existing jxr command to an existing ring to have a similar behavior
// e.g. drop jxr on ring, bind them so that next time an element goes through, it's applied via binded ring
elCode.setAttribute("anchor", "left")
elCode.setAttribute("target", "")
elCode.setAttribute("value", code)
elCode.setAttribute("position", ".0 "+codeVerticalOffset+" .0")
elCode.setAttribute("scale", "0.1 0.1 0.1")
el.appendChild( elCode )
active_rings.appendChild( el )
return el
}
function endRingCheck(){
clearInterval(ringCheckInterval)
}
function applyNextFilterInteraction (){}
let sequentialFiltersInteractionOnPicked = []
let sequentialFiltersInteractionOnReleased = []
let currentFilterOnPicked = null
let currentFilterOnReleased = null
</script>
</head>
<body style="background:linear-gradient( 45deg, #324, #8c74ff)">
<a-scene selfcontainer
light="defaultLightsEnabled: true"
cursor="rayOrigin: mouse"
renderer="colorManagement: false; stencil: true; antialias:true; highRefreshRate:true; foveationLevel: 0.5;"
xr-mode-ui="XRMode: xr"
obb-collider="showColliders:false"
raycaster="objects: [html]; interval:100;"
>
<a-sphere id=contextsphere wireframe=true position="0 1.4 -1" radius=.3></a-sphere>
<a-console position="-1 1.3 0" rotation="-45 90 0" font-size="34" height="1" skip-intro="true"></a-console>
<a-entity isoterminal="iso: https://forgejo.isvery.ninja/assets/xrsh-buildroot/main/xrsh.iso; minimized:true; overlayfs: /package.overlayfs.zip; bootMenu: 1; bootMenuURL: 1" position="0 1.6 -0.3">
<a-entity launcher pressable position="0 -0.315 0"></a-entity>
</a-entity>
<a-sphere position="1 1 -2"></a-sphere>
<a-entity helloworld-window pressable position="0 1.6 0"></a-entity>
<a-entity id="rig">
<a-entity id="player">
<a-entity camera="fov:90" position="0 1.6 0" id="camera" wasd-controls gaze-touch-to-click></a-entity>
<a-entity id="left-hand" hand-tracking-grab-controls="hand: left; modelColor: #EEEEEE; hoverEnabled:true"
laser-controls="hand: left"
raycaster="far:0.04"
blink-controls="cameraRig:#player; teleportOrigin: #camera; collisionEntities: #floor"
pinchsecondary
>
</a-entity>
<a-entity id="right-hand" hand-tracking-grab-controls="hand: right; modelColor: #EEEEEE; hoverEnabled:true"
laser-controls="hand: right"
raycaster="far:0.04"
pinchprimary
blink-controls="cameraRig:#player; teleportOrigin: #camera; collisionEntities: #floor"
____pinch-to-teleport="rig: #player"></a-entity>
</a-entity>
</a-entity>
<a-sphere segments-width=12 segments-height=12 pressable="" start-on-press="" id="box" radius="0.033" color="gray"></a-sphere>
<a-box pressable="" start-on-press-other="" id="otherbox" scale=".05 .05 .05" opacity=.3 wireframe=false color="white"></a-box>
<a-entity id=rings>
<!-- consider applyNextFilterInteraction() -->
<!-- sequence of rings, namely no "absolute" list, in addition to apply a function to the content currently selected, it opens up the "next" rings and stacks itself to the applied rings -->
<a-troika-text anchor=left target value= "jxr addKeyboardRings()" position="0 1.10 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text anchor=left target value= 'jxr Array.from( active_rings.querySelectorAll("a-torus") ).map( el => { el.setAttribute("radius", ".01"); el.setAttribute("radius-tubular", ".001") })' position="0.4 1.20 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value= "jxr sceneIdsToElementForRings()" position="0 1.20 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='example text' position="0 1.30 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='a-sky' position="0 1.40 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='.collidable' position="0 1.60 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-troika-text onpicked="startRingCheck()" onreleased="endRingCheck()" anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position="0 1.50 -.8" scale="0.1 0.1 0.1"></a-troika-text>
<a-entity id=active_rings>
<a-torus color="green" position="0.4 1.4 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr isoterminal("ls")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="green" position="0.4 1.4 -.8" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr isoterminal(" | wc -l")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="blue" position="0 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="white" position="0 1 -.3" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToWhite")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="white" position=".5 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr scaleUp()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="white" position="-.5 1 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr addRingFromCode()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus color="white" position="-.5 1.5 -.5" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr addToRingSelection()' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
</a-entity>
<!-- see history of transformations via voice, and cancelling it, older PoC -->
<a-entity id=rings_stack target position="-1 1 -1.5">
<a-troika-text anchor=left target value='previous transformations' position="-.1 .3 .1" scale="0.1 0.1 0.1"></a-troika-text>
<a-torus wireframe=true color="#43A367" position="0 0 -.1" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text curve-radius=1 rotation="90 0 0" anchor=left target value='jxr applyFunctionToSelection("changeColorToRed")' position=".0 .1 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
<a-torus wireframe=true color="#43A367" position="0 0 0" segments-radial=4 segments-tubular=12 opacity=.3 radius=".1" radius-tubular="0.01">
<a-troika-text anchor=left target value='jxr applyFunctionToSelection("changeColorToBlue")' position=".0 .15 .0" scale="0.1 0.1 0.1"></a-troika-text>
</a-torus>
</a-entity>
</a-entity>
</a-scene>
</body>
<script>
const term=document.querySelector('[isoterminal]');
term.emit("exec", ["ls -l", (str,buf) => console.log(str) ])
// HOWDY DEVELOPER!
// for a REPL boilerplate see this codepen instead: https://codepen.io/coderofsalvation/pen/wBwRpew
document.querySelector('a-scene').addEventListener('isoterminal_init', function(e){
const term=document.querySelector('[isoterminal]');
term.addEventListener('ready', e => {
term.emit("exec", ["ls -l", (str,buf) => console.log(str) ])
})
// insert rings here instead
// example custom REPL (https://xrsh.isvery.ninja/#Custom%20REPLs)
const myREPL = {
key: "f",
prompt: "\r\nmyrepl> ",
title: (opts) => `other awesome NLnet FOSS projects ❤️`,
init: function( mainmenu ){
window.open('https://nlnet.nl/project/current.html','_blank')
this.send("\n\rreturning to mainmenu")
mainmenu()
},
keyHandler: function(ch){
this.send(ch) // echo keys to user
},
cmdHandler: function(cmd){
this.send("\r\n"+cmd) // only works after commenting mainmenu()\n")
this.send(myREPL.prompt)
}
}
ISOTerminal.prototype.boot.menu.push(myREPL)
})
// consider bringing https://companion.benetou.fr/gesture-exploration.js directly here instead
setTimeout( _ => {
let jointTestEl1 = addNewNote( "jxr document.querySelector('a-sky').setAttribute('color','purple')", "0 .03 -.05")
jointTestEl1.setAttribute("scale", ".05 .05 .05")
jointTestEl1.setAttribute("rotation", "0 -90 0")
jointTestEl1.setAttribute("target", "")
jointTestEl1.id = "jointtest1"
let jointTestEl2 = addNewNote( "jxr document.querySelector('a-sky').setAttribute('color','gray')", "0 .03 -.05")
jointTestEl2.setAttribute("scale", ".05 .05 .05")
jointTestEl2.setAttribute("position", "0 .03 -.05")
jointTestEl2.setAttribute("rotation", "0 -90 0")
jointTestEl2.id = "jointtest2"
AFRAME.scenes[0].setAttribute("bind-element-to-finger__test1", { hand: 'r_handMeshNode', finger: 'index-finger-tip', target: '#jointtest1' })
AFRAME.scenes[0].setAttribute("bind-element-to-finger__test2", { hand: 'r_handMeshNode', finger: 'thumb-tip', target: '#jointtest2' } )
}, 1000)
AFRAME.registerComponent('bind-element-to-finger', {
multiple: true,
schema: {
hand: {type: 'string', default: 'r_handMeshNode'},
finger: {type: 'string', default: 'index-finger-tip'},
target : {type: 'selector'},
},
init: function () {
},
tick: function (time, timeDelta) {
const joint = AFRAME.scenes[0].object3D.getObjectByName(this.data.hand)?.parent.getObjectByName(this.data.finger)
if ( joint && this.data.target.object3D.parent == AFRAME.scenes[0].object3D ) this.data.target.object3D.parent = joint
// unfortunately reparenting for now breaks properly picking...
// could check in core for whom the parent is and reparent only on release
// or potentially not at all, maybe the user expects to be able to pick the instruction away from the hand
// would also then need an explicit mechanism to bind back
// a la wrist shortcut
// warning that some parenting is done "right" namely by removing the offset then become an A-Frame child with that offset
// allowing thus the expected behavior in that context
// consequently might have to distinguish the 2 situations
}
})
/* // does not work reliably, seems to be just the 1st keypress
var keyEvent = document.createEvent('KeyboardEvent');
keyEvent.initKeyboardEvent('keydown', true, false, null, 0, false, 0, false, 77, 0);
window.dispatchEvent(keyEvent);
*/
</script>
</html>

1002
src/jxr-core.js Normal file

File diff suppressed because it is too large Load diff

535
src/uglyph.js Normal file
View file

@ -0,0 +1,535 @@
/* potential improvements
event on state switch, e.g. thumb up to thumb down or whatever to thumb down
but NOT thumb down to thumb down
sustained state, e.g. thumb down to thumb down for N seconds
extend proximityBetweenJointsCheck to any object3D or from 1 object3D to a class of entities (which themselves are object3D)
generalize showGestureDebug to any joint, not just thumb-tip of right hand
*/
// consider which gestures are more inclusive, more minimalist to be more accessible
let microgestures = []
microgestures.push( {"microgesture":{"type":"glyph","action":"Extension","context":["Contact","Air"],"parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["thumb"]],"phalanx":[]}} )
// supports both hands
microgestures.push( {"microgesture":{"type":"seq","quantificators":{"x":[[["thumb"],["index"],["middle"],["ring"],["pinky"]],[["thumb"],["index"],["middle"],["ring"],["pinky"]]],"y":[]},"seq":{"type":"and","operands":[{"type":"glyph","action":"Any","context":["Air","Contact"],"parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x0"]],"contact":{"type":"contact","action":"Contact","parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x1"]]},"phalanx":[]},{"type":"glyph","action":"Any","context":["Air","Contact"],"parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x1"]],"contact":{"type":"contact","action":"Contact","parameters":{"pressure":{"start":null,"end":null,"type":"no_end"},"amplitude":{"start":null,"end":null,"type":"no_end"},"time":{"start":null,"end":null,"leftType":"none","rightType":"bigger"}},"actuator":[["x0"]]},"phalanx":[]}]}}} )
// name = closed gap touch
// text description = Press the specified fingers together.
// different from "finger pinch" which is just between thumb and another finger, at specified locations
// from https://lig-microglyph.imag.fr
// unfortunately limited reach https://scholar.google.com/scholar?cites=15369904842495978041
// no known alternative as of July 2025, so impact limited to HCI researchers, not industry or developers
// example of unpacking uGlyph LaTeX representation in JSON format
// microgestures[0].microgesture.actuator.map( e => e.map(a => console.log(a)) )
// if all actuator / quantificators in allJointsNames potential check
// filter on contact based on e.g. proximityBetweenJointsCheck(joints)
// based on Extension
// Air interpreted as no contact
// unsupported by WebXR (or even OpenXR)
// pressure beyond binary on/off
// unknown actuators, e.g. x0
// contact beyond joints, even though some might be infered,
// e.g. middle of palm as center between metacarpal and wrist
// high precision contact beyond thumb/index
// limited due to privacy concerns by browser vendors
const fingersNames = ["index-finger", "middle-finger", "ring-finger", "pinky-finger","thumb"]
const tips = fingersNames.map( f => f+"-tip" )
const thumbParts = ["metacarpal", "phalanx-proximal", "phalanx-distal"] // no phalanx-intermediate for thumb
const fingerParts = thumbParts.concat(["phalanx-intermediate"])
const fingers = tips.concat( thumbParts.map( f => fingersNames.at(-1)+"-"+f ), fingerParts.flatMap( fp => fingersNames.slice(0,4).map( fn => fn+"-"+fp) ) )
const allJointsNames = ["wrist"].concat( fingers ) // also has wrist, no fingers
// console.log( allJointsNames.sort() )
function shortVec3(v){ return {x:v.x.toFixed(3), y:v.y.toFixed(3), z:v.z.toFixed(3)} } ;
// assumes joints, could be generalized to any Object3D
function proximityBetweenJointsCheck(joints){
const thresholdDistance = .008
// contacts even while hands resting
// 2cm : 8
// 1cm : 4
// 9mm : 2
// 8mm : 0 ... but also prevents some contacts, e.g. finger tips accross fingers
// consequently would have to identify which contacts take place at rest
// might be from within same finger and thus potentially to filter out when "next" to each other joint
// e.g. finger tip could physiologically touch own metacarpal but no phalanx
// BUT it can for the same finger on the other hand
let contacts = []
let combinations = joints.flatMap( (v, i) => joints.slice(i+1).map( w => [v, w] ))
// from https://stackoverflow.com/a/43241287/1442164
combinations.map( j => {
let rt = j[0].position
let lt = j[1].position
//console.log( 'checking: ', rt, lt )
let dist = rt.distanceTo(lt)
if ( dist < thresholdDistance ) {
contacts.push( {dist: dist.toFixed(3), a:j[0].name, ah: j[0].parent.parent.el.id, b:j[1].name, bh: j[1].parent.parent.el.id } )
// assumes a bone, could check first on type, otherwise can have different behavior
// could add the timestamp and position value at that moment
}
})
return contacts
// getting up to 45 contacts checking 5 finger tips on each hand, which is correct for C10,2
}
// could also attach the value then show next to the joint
let debugValue = {}
function addDebbugGraph(){
el = document.createElement("a-box")
el.id = "debuggraph"
el.setAttribute("scale", "1 .3 .01")
el.setAttribute("position", "0 1.4 -1")
AFRAME.scenes[0].appendChild(el)
}
// used an array of points, e.g. pos.x over time, thus every 50ms xTimeSeries.push(pos.x)
function drawPoints(points){
if (debugValue.length<10) return
let canvas = document.createElement('canvas');
canvas.width = 1000;
canvas.height = 100 * Object.values( points).length
const ctx = canvas.getContext("2d")
ctx.fillStyle = "white";
ctx.fillRect(0, 0, canvas.width, canvas.height);
// might want to append (and thus track status) in in order to show the result live
// or "just" take the last 10 elements of array
// middle should be 0... as we can go negative on that axis
//points.slice(-10).map( (p,n) => {
let verticalOffsetSize = 50
Object.values( points).map( (v,i) => {
ctx.beginPath()
ctx.moveTo(0, 0)
let values = v
if (v.length > 100) values = v.slice(-100)
ctx.strokeStyle = "black";
values.map( (p,n) => {
let value = Math.floor( 100-1+p*100 )
ctx.lineTo(n*10, value+i*verticalOffsetSize)
ctx.moveTo(n*10, value+i*verticalOffsetSize)
if (value>100-10 && value<100+10) {
console.log('customgesture', value)
AFRAME.scenes[0].emit('customgesture')
ctx.strokeStyle = "green";
}
})
ctx.stroke()
})
ctx.beginPath()
ctx.moveTo(0, 100-10)
ctx.lineTo(canvas.width, 100-10)
ctx.moveTo(0, 100+10)
ctx.lineTo(canvas.width, 100+10)
ctx.strokeStyle = "red";
ctx.stroke()
let el = document.getElementById("debuggraph")
el.setAttribute("src", canvas.toDataURL() ) // somehow works on other canvas...
// el.object3D.children[0].material.needsUpdate = true
//console.log( el.src ) // works but does not update
return el
}
// gesture(s) to detect, callback on detection, by default debugging via console or new debuggraph
// should also emit event
AFRAME.registerComponent('gesture-listeners', {
// WARNING : console.log is very costly as detectors are very frequent, e.g. every 50ms
// spamming the console is NOT recommended in XR and otherwise
// consider alternative visual results, e.g. changing the color of an entity
events: {
gesture : function (event) {
console.log( 'gesture captured:', event.detail )
},
gesturecontext : function (event) {
console.log( 'gesture context captured:', event.detail )
}
}
})
// might need some form of hierarchy, so that composable gestures don't overlap or re-use components from other gestures
// e.g. if there is a fingertip to metacarpal gesture for the index, don't re-write the same for the little finger
// same for other hand, i.e. left vs right
// could be computationally interesting too, e.g. stop checking for a gesture if part of it failed already, e.g. stop checking for a gesture if part of it failed already
// suggestion for architecture
AFRAME.registerComponent('template-gesture-detector', {
schema: {
threshold: {type: 'number', default: .3}, // arbitrary unit, e.g distance in m, sum of angles in degrees, etc
frequency: {type: 'number', default: 50}, // in ms
target: {type: 'selector'}, // common way to apply the transformation to the result
callback: {type: 'string', default: ''},
},
init: function () {
this.tick = AFRAME.utils.throttleTick(this.tick, this.data.frequency, this);
let target = this.data.target
let callback = this.data.callback
AFRAME.scenes[0].addEventListener('gesture', event => {
if (event.detail.type == 'template-gesture-detector'+'_gesture') { // filtering for this specific gesture
if (target) target.setAttribute("position", AFRAME.utils.coordinates.stringify( event.detail.targetPosition ) )
// optional callback
if (callback) window[callback]( event )
}
} )
},
tick: function (t, dt) {
const myScene = AFRAME.scenes[0].object3D // optional simplification
// condition to start detecting, e.g. here right hand detected
if (!myScene.getObjectByName("r_handMeshNode") ) return
const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist") // optional simplification
// detector itself
let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z)
// where helpers e.g. allJointsNames[] with distanceTo() on position can be used or proximityBetweenJointsCheck(joints)
// condition to emit
if ( sum < this.data.threshold ) {
AFRAME.scenes[0].emit('gesture', {
date:Date.now(),
type: 'template-gesture-detector'+'_gesture',
targetPosition:wrist.position, // passing on data specific to detector
value:sum
})
}
}
})
AFRAME.registerComponent('cube-pull', {
schema: {
threshold: {type: 'int', default: .3},
},
init: function () {
this.tick = AFRAME.utils.throttleTick(this.tick, 50, this);
},
tick: function (t, dt) {
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("l_handMeshNode") ) return
const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist")
// detector itself
let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z)
// console.log( sum )
if ( sum < this.data.threshold ) {
// doesn't look good, cube on wrist is moving quite a bit too
// could emit event too
AFRAME.scenes[0].emit('gesture', {date:Date.now(), type:'cube pull', value:sum})
// would be better with gesture detector type, not how it it used
// effect should be delegated to event listener instead
cubetest.setAttribute("position", AFRAME.utils.coordinates.stringify( wrist.position ) )
// could check if all joints have close to 0 rotation on ...
// are roughly on the same y-plane of the wrist (facing up or down)
}
}
})
AFRAME.registerComponent('verticalflatpalmrighthand', {
schema: {
command: {type: 'string'},
},
init: function () {
this.tick = AFRAME.utils.throttleTick(this.tick, 50, this);
window.lastGesture = Date.now() // to initialize
},
tick: function (t, dt) {
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("l_handMeshNode") ) return
const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist")
let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z)
// console.log( sum )
//document.querySelector('[isoterminal]').emit("send", [sum + (window.lastGesture+1000>now)]) // +"\n"]) // problematic for special commands
//document.querySelector('[isoterminal]').emit("send", ["\n"]) // kind of horizontal
if ( sum < .3 ) {
// spams the terminal
// could emit event too
let now = Date.now()
if ( now - window.lastGesture > 1000 ){
document.querySelector('[isoterminal]').emit("send", [this.data.command]) // +"\n"]) // problematic for special commands
window.lastGesture = now
AFRAME.scenes[0].emit('gesture', {date:now, type:'vertical flat palm', value:sum})
}
// no event if inhibited due to refractory period
}
if ( sum > 1 ) {
let now = Date.now()
if ( now - window.lastGesture > 1000 ){
document.querySelector('[isoterminal]').emit("send", ["\n"]) // kind of horizontal
window.lastGesture = now
AFRAME.scenes[0].emit('gesture', {date:now, type:'horizontal flat palm', value:sum})
}
// no event if inhibited due to refractory period
}
// could check if all joints have close to 0 rotation on ...
// are roughly on the same y-plane of the wrist (facing up or down)
}
})
AFRAME.registerComponent('gestures', {
init: function () {
this.tick = AFRAME.utils.throttleTick(this.tick, 50, this);
// maybe every 50ms is too often, or not often enough, to check
},
// consider also tock, cf https://aframe.io/docs/1.7.0/core/component.html#before-after
tick: function (t, dt) {
// consider here the order or hierarchy
// optimisation? reconciliation? cascading?
// example of context, e.g. using distance of wrist to a target object, or time, or any other condition
// could be defined else as long as it respect a specific format, like filters/converters/etc
// starting with emiting an event, e.g.
// AFRAME.scenes[0].emit('gesture', {date:now, type:'horizontal flat palm', value:sum})
// or context (unsure if truly has to be separated)
// AFRAME.scenes[0].emit('gesturecontext', {date:now, type:'right wrist in volume of target' })
// detect gesture A
// emit event
// detect gesture B
// emit event
// detect gesture C
// emit event
},
})
AFRAME.registerComponent('positional-context', {
schema: {
target: {type: 'selector'},
attribute: {type: 'string'},
value: {type: 'string'},
threshold: {type: 'float'}
},
init: function () {
this.tick = AFRAME.utils.throttleTick(this.tick, 500, this);
},
tick: function (t, dt) {
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("l_handMeshNode") ) return
const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist").position
// console.log ( this.data.target.object3D.position, this.data.threshold, wrist.distanceTo( this.data.target.object3D.position ) < this.data.threshold )
if ( wrist.distanceTo( this.data.target.object3D.position ) < this.data.threshold ){
let now = Date.now()
// could emit event too
AFRAME.scenes[0].setAttribute(this.data.attribute, "command:"+ this.data.value)
// wrong... this should still delegate instead to a gesture (or gestures) that would THEN conditionally pass the command
AFRAME.scenes[0].emit('gesturecontext', {date:now, type:'right wrist in volume of target' })
// could be used to check for enter/leave based on previous state until now
} else {
AFRAME.scenes[0].removeAttribute(this.data.attribute)
}
}
})
// detectors
function pullCubeToWrist(){
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("l_handMeshNode") ) return
const wrist = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("wrist")
let sum = Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.y) + Math.abs(wrist.rotation.z)
console.log( sum )
if ( sum < .3 ) cubetest.setAttribute("position") = wrist.position // doesn't look good, cube on wrist is moving quite a bit too
// could check if all joints have close to 0 rotation on ...
// are roughly on the same y-plane of the wrist (facing up or down)
}
function gestureThumbEndingAnyContact (){
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("l_handMeshNode") ) return
// potential shortcuts :
const leftHandJoints = myScene.getObjectByName("l_handMeshNode").parent.children.filter( e => e.type == "Bone")
const rightHandJoints = myScene.getObjectByName("r_handMeshNode").parent.children.filter( e => e.type == "Bone")
const allHandsJoints = leftHandJoints.concat( rightHandJoints )
let posA = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position
let contactPointsToThumbA = leftHandJoints
.concat( rightHandJoints[0].parent.children.filter( e => e.name != 'thumb-tip' ) ) // right hand except thumb tip
.map( e => e.position.distanceTo(posA) ).filter( d => d < .02 ) // threshold of 2cm distance (must be bigger than thumb-tip to previous join position)
// relatively compact description and maybe relatively computively cheap
let pos = myScene.getObjectByName("l_handMeshNode").parent.getObjectByName("thumb-tip").position
let contactPointsToThumb = rightHandJoints
.concat( leftHandJoints[0].parent.children.filter( e => e.name != 'thumb-tip' ) ) // right hand except thumb tip
.map( e => e.position.distanceTo(pos) ).filter( d => d < .02 ) // threshold of 2cm distance (must be bigger than thumb-tip to previous join position)
if (contactPointsToThumb.length+contactPointsToThumbA.length < 1) console.log('no contact'); else console.log('thumb tip in contact with same hand or other hand')
// on contact could also return the join number/names
}
function testAvegageValue (){
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("r_handMeshNode") ) return
let rt = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip");
debugValue.x.push( rt.position.x )
let v = debugValue.x
const windowSize = 10 // otherwise too long, e.g 100x500ms gives 5s average
if (v.length > windowSize) {
values = v.slice(-windowSize)
let avg = ( values.reduce( (acc,c) => acc+c )/windowSize) .toFixed(3)
console.log( avg )
}
}
function showContactPoints (){
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("r_handMeshNode") ) return
let targetJoints = []
tips.map( t => targetJoints.push( myScene.getObjectByName("r_handMeshNode").parent.getObjectByName(t) ) )
tips.map( t => targetJoints.push( myScene.getObjectByName("l_handMeshNode").parent.getObjectByName(t) ) )
// tips only
let contacts = proximityBetweenJointsCheck(targetJoints)
if (contacts.length) {
console.log( "contacts:", contacts )
// {dist: dist.toFixed(3), a:j[0].name, ah: j[0].parent.parent.el.id, b:j[1].name, bh: j[1].parent.parent.el.id }
contacts.map( c => {
// show value or even just a temporary object there
let a = document.getElementById(c.ah).object3D.getObjectByName(c.a)
let b = document.getElementById(c.bh).object3D.getObjectByName(c.b)
const geometry = new THREE.BoxGeometry( .01, .01, .01 )
const material = new THREE.MeshBasicMaterial( { color: 0x00ff00 } )
const cube = new THREE.Mesh( geometry, material )
a.add( cube )
})
}
}
function showGestureDistanceDebugJoints(){
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("r_handMeshNode") ) return
let targetJoints = []
tips.map( t => targetJoints.push( myScene.getObjectByName("r_handMeshNode").parent.getObjectByName(t) ) )
tips.map( t => targetJoints.push( myScene.getObjectByName("l_handMeshNode").parent.getObjectByName(t) ) )
// console.log( targetJoints ) looks fine
//console.log( "contacts:", proximityBetweenJointsCheck(targetJoints)
// tips only
let targetJointsFull = []
allJointsNames.map( t => targetJointsFull.push( myScene.getObjectByName("r_handMeshNode").parent.getObjectByName(t) ) )
allJointsNames.map( t => targetJointsFull.push( myScene.getObjectByName("l_handMeshNode").parent.getObjectByName(t) ) )
let contacts = proximityBetweenJointsCheck(targetJointsFull)
if (contacts.length) console.log( "contacts:", contacts )
}
function showGestureDistanceDebug(){
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("r_handMeshNode") ) return
let rt = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip").position
let lt = myScene.getObjectByName("l_handMeshNode").parent.getObjectByName("thumb-tip").position
if ( rt.distanceTo(lt) < .1 )
console.log( 'lt close to rt')
else
console.log( rt.distanceTo(lt) )
}
function showGestureDebug(){
const myScene = AFRAME.scenes[0].object3D
if (!myScene.getObjectByName("r_handMeshNode") ) return
let rt = myScene.getObjectByName("r_handMeshNode").parent.getObjectByName("thumb-tip");
//console.log( shortVec3( rt.position ), shortVec3( rt.rotation ) )
// could do for the 2x25 values... but then becomes unreadible, hence why showing sparklines could help
// can be done on HUD
if (!debugValue.x){
debugValue.x = []
debugValue.y = []
debugValue.z = []
debugValue.a = []
debugValue.b = []
debugValue.c = []
}
debugValue.x.push( rt.position.x )
debugValue.y.push( rt.position.y )
debugValue.z.push( rt.position.z )
debugValue.a.push( rt.rotation.x )
debugValue.b.push( rt.rotation.y )
debugValue.c.push( rt.rotation.z )
let el = document.getElementById("debuggraph")
if (!el) addDebbugGraph()
drawPoints( debugValue )
}
let detectors = []
detectors.push( gestureThumbEndingAnyContact )
detectors.push( pullCubeToWrist )
// debugging tools
detectors.push( testAvegageValue )
detectors.push( showContactPoints )
detectors.push( showGestureDistanceDebugJoints )
detectors.push( showGestureDistanceDebug )
detectors.push( showGestureDebug )
detectors.push( secondaryPinchSingleHanded )
// usable as e.g. setInterval( i => { pullCubeToWrist() }, 500 )
// but would be better used a AFrame component with AFRAME.utils.throttleTick()
// note that though this does not insure hand tracking is ready yet, hence testing on ThreeJS object r/l of _handMeshNode
AFRAME.registerComponent('bind-element-to-finger', {
multiple: true,
schema: {
hand: {type: 'string', default: 'r_handMeshNode'},
finger: {type: 'string', default: 'index-finger-tip'},
target : {type: 'selector'},
},
tick: function (time, timeDelta) {
const joint = AFRAME.scenes[0].object3D.getObjectByName(this.data.hand)?.parent.getObjectByName(this.data.finger)
if ( joint && this.data.target.object3D.parent == AFRAME.scenes[0].object3D ) this.data.target.object3D.parent = joint
}
})
const primaryPinchSingleHandedDistanceThreshold = 0.02 // 2 cm
const secondaryPinchSingleHandedDistanceThreshold = 0.05 // need to be much looser, maybe due to privacy limitations or camera position (hidden fingers)
function secondaryPinchSingleHanded (){
if ( !AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode") ) return
let j1 = AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode")?.parent.getObjectByName("thumb-tip").position
let j2 = AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode")?.parent.getObjectByName("index-finger-tip").position
let j3 = AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode")?.parent.getObjectByName("pinky-finger-tip").position
let j4 = AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode")?.parent.getObjectByName("middle-finger-tip").position
let d1 = j1.distanceTo( j2 )
let d2 = j1.distanceTo( j3 )
let d3 = j1.distanceTo( j4 )
// console.log( 'normal pinch distance', d1 )
// console.log( 'secondary pinch distance', d2 )
// document.querySelector("a-sky").setAttribute("color", "gray")
if ( d1 < primaryPinchSingleHandedDistanceThreshold && d2 > secondaryPinchSingleHandedDistanceThreshold )
AFRAME.scenes[0].emit('secondary-singled-handed-pinch-not', {hand: 'right'})
// console.log( 'normal pinch' ) // just for testing
if ( d1 < primaryPinchSingleHandedDistanceThreshold && d2 < secondaryPinchSingleHandedDistanceThreshold )
AFRAME.scenes[0].emit('secondary-singled-handed-pinch', {hand: 'right'})
if ( d1 < primaryPinchSingleHandedDistanceThreshold && d3 < primaryPinchSingleHandedDistanceThreshold )
AFRAME.scenes[0].emit('secondary-singled-handed-pinch-middle-finger', {hand: 'right'})
}
/* usage examples
AFRAME.scenes[0].addEventListener('secondary-singled-handed-pinch', evt => {
//addNewNote('added here...', AFRAME.utils.coordinates.stringify( pinches.filter( p => p.primary ).at(-1).position ) )
} )
let lastSecondaryPinchExecuted = Date.now()
AFRAME.scenes[0].addEventListener('secondary-singled-handed-pinch-middle-finger', evt => {
if ( Date.now() - lastSecondaryPinchExecuted < 500 ){
// console.warn('ignoring, executed secondary pinch during the last 500ms already')
let x = 42 // added only to comment the previous line
} else {
lastSecondaryPinchExecuted = Date.now()
//console.log('mid pinched after 500ms...')
let el = addNewNote('note', AFRAME.utils.coordinates.stringify( AFRAME.scenes[0].object3D.getObjectByName("r_handMeshNode")?.parent.getObjectByName("thumb-tip").position ) )
el.setAttribute("troika-text", "outlineWidth", 0)
setTimeout( _ => el.object3D.lookAt( player.object3D.position ), 500 )
}
} )
*/

71
src/uglyph_tests.js Normal file
View file

@ -0,0 +1,71 @@
function testCubePull(){
AFRAME.scenes[0].setAttribute("cube-pull", "")
// multi-users as multi-gestures?
}
// cascading gestures
function testVerticalFlatRightHand(){
let el = document.createElement("a-sphere")
el.id = "contextsphere"
el.setAttribute("radius", "0.3") // fails on .3 or 0.3
//el.setAttribute("scale", "0.3)
el.setAttribute("wireframe", "true")
el.setAttribute("position", "0 1.4 -1")
AFRAME.scenes[0].appendChild(el)
let cube = document.createElement("a-box")
cube.setAttribute("target", "") // no effect, probably over written by XRSH
cube.setAttribute("color", "green")
cube.id = "cubetest"
cube.setAttribute("scale", ".1 .1 .1")
cube.setAttribute("position", "0 1.4 -1")
AFRAME.scenes[0].appendChild(cube)
AFRAME.scenes[0].setAttribute("positional-context", "target:#contextsphere; threshold:0.3; attribute:verticalflatpalmrighthand; value: ls -l\\n;")
AFRAME.scenes[0].setAttribute('gesture-listeners','')
// gesture to
// go back in history (then down again)
// to execute
/*
// enter
document.querySelector('[isoterminal]').emit("send", ["\n"])
// arrow keyup
document.querySelector('[isoterminal]').emit("send", ["\x1b[A"])
// arrow keydown
document.querySelector('[isoterminal]').emit("send", ["\x1b[B"])
*/
// that can also be visible, e.g. wireframe
}
function testCubePullWithinContext(){
// let cube = addCubeWithAnimations() // defined in JXR core probably rather in example
// removing dependencies for testing
let cube = document.createElement("a-box")
cube.setAttribute("wireframe", "true")
cube.setAttribute("color", "red")
cube.setAttribute("scale", ".1 .1 .1")
cube.setAttribute("position", "0 1.4 -1")
cube.setAttribute("target", "")
cube.id = "cubetest"
//cube.setAttribute("visible", "false")
AFRAME.scenes[0].appendChild(cube)
el = document.createElement("a-sphere")
el.id = "contextsphere"
el.setAttribute("radius", .3)
el.setAttribute("wireframe", "true")
el.setAttribute("position", "0 1.4 -1")
AFRAME.scenes[0].appendChild(el)
AFRAME.scenes[0].setAttribute("positional-context", "target:#contextsphere; threshold:0.3; attribute:cube-pull;")
// that can also be visible, e.g. wireframe
// could do so with example of index finger tip within range of targets, if so make a visual change
// on within cube, off if outside of it
// could change the cube color to show the difference too
// try perf on Quest1
}