From 930b2705381b4aa3e4cb7c7a7329cc509ab06b28 Mon Sep 17 00:00:00 2001 From: Leon van Kammen Date: Mon, 4 Aug 2025 18:06:41 +0200 Subject: [PATCH] milestone 4g: manyfold.sh auto-generate experience.zip index.json on change (inotify) --- manyfold/README.md | 21 ++++++++ manyfold/cli/manyfold.sh | 54 +++++++++++-------- manyfold/root/.config/rclone/rclone.conf | 10 ++++ manyfold/root/hook.d/boot/placeholder.sh | 1 + .../root/hook.d/daily/delete_big_files.sh | 19 +++++++ manyfold/root/hook.d/hourly/placeholder.sh | 2 + .../root/hook.d/inotify_CREATE/placeholder.sh | 1 + .../inotify_MODIFY/package_experience.sh | 5 ++ .../root/hook.d/inotify_MODIFY/placeholder.sh | 1 + manyfold/root/hook.d/inotify_MOVED_TO | 1 + manyfold/xrforge.landscape.svg | 5 ++ nix/docker.nix | 3 ++ 12 files changed, 101 insertions(+), 22 deletions(-) create mode 100644 manyfold/root/.config/rclone/rclone.conf create mode 120000 manyfold/root/hook.d/boot/placeholder.sh create mode 100755 manyfold/root/hook.d/daily/delete_big_files.sh create mode 100755 manyfold/root/hook.d/hourly/placeholder.sh create mode 120000 manyfold/root/hook.d/inotify_CREATE/placeholder.sh create mode 100755 manyfold/root/hook.d/inotify_MODIFY/package_experience.sh create mode 120000 manyfold/root/hook.d/inotify_MODIFY/placeholder.sh create mode 120000 manyfold/root/hook.d/inotify_MOVED_TO create mode 100644 manyfold/xrforge.landscape.svg diff --git a/manyfold/README.md b/manyfold/README.md index 95e338b..4e1301a 100644 --- a/manyfold/README.md +++ b/manyfold/README.md @@ -19,6 +19,7 @@ $ docker load < $(nix-build nix/docker.nix) | `THEME` | `default` | bootstrap theme | | `NO_OVERLAYFS` | `` | disable the filesystem overlay mechanism | | `NO_DEFAULTDB` | `` | disable the default db (activates manyfold installer) | +| `NO_DELETEBIGFILES` | `` | disable deleting big files which are older than 5 days and bigger than ($currentyear-2020) MB's | | `RCLONE_REMOTE` | `` | specify **single** rclone remote name (without semicolon) to mount | | `UPLOAD_PATH` | `/mnt/models`| specify default library where user-files are uploaded (regular dir or mounted rclone path) | @@ -54,3 +55,23 @@ Your drives will get automagically mounted and added to the database automagical TIP: use **alphanumeric** names for rclone remotes (manyfold libraries choke on dot- or other special-characters) +# Unixy event hooks + +Until WebEvents [will get implemented on a REST-level in manyfold](https://github.com/orgs/manyfold3d/projects/4/views/1?filterQuery=Pub&pane=issue&itemId=108834509&issue=manyfold3d%7Cmanyfold%7C4097) Things like boot-phase, scheduler and file-changes can be reacted up via the `/root/hook.d` directory: + +``` +$ ls /root/hook.d +boot +daily +hourly +inotify_CREATE +inotify_MODIFY +``` + +You can put scripts in there, which are fired when needed. + +> Example: [manyfold/root/hook.d/daily/delete_big_files.sh] is triggered daily to cleanup files which exceed a certain age/size. + +Currently inotify events (`inotify_MODIFY` e.g.) are triggered when files in `/mnt/models` change. + +> Perhaps in the future this will also work for rclone remotes, by writing a `hourly`-script which scans them and fires `inotify_MODIFY` accordingly. diff --git a/manyfold/cli/manyfold.sh b/manyfold/cli/manyfold.sh index 8879489..9659a5d 100755 --- a/manyfold/cli/manyfold.sh +++ b/manyfold/cli/manyfold.sh @@ -56,6 +56,31 @@ infinite(){ loop "$@" } +# flexible unixy hook-mechanism which executes all files in ~/hook.d/foo/* when +# calling 'hook foo bar' +hook(){ + test -z "$1" && { echo "usage: hook [args]"; return 0; } + logger "$ hook $*" + cmd=$1 + shift + test -d ~/hook.d/$cmd && { + find -L ~/hook.d/$cmd/ -type f -executable -maxdepth 1 | while read hook; do + logger " |+ hook $hook $*" + { $hook "$@" || true; } 2>&1 | awk '{ gsub(/\/root\/\//,"",$1); $1 = sprintf("%-40s", $1)} 1' | logger + done + } +} + +start_hook_daemon(){ + # every day call scripts in ~/hook.d/cleanup_daily/* + # 86400 secs = 1 day 3600 = 1 hour + $0 infinite 86400 hook daily & + $0 infinite 3600 hook hourly & + # trigger hooks when files change in /mnt/models + inotifywait -r -m /mnt/models | awk '$2 ~ /(CREATE|MODIFY|MOVED_TO)/ { system("'$0' hook inotify_"$2" "$1""$3) }' & +} + + db(){ default(){ test -f /manyfold/manyfold.sql && ! test -f /config/manyfold.sql && { @@ -117,7 +142,7 @@ set_theme(){ set_modelpath(){ echocolor "[$APPNAME]" "enforcing modelpath" - debug sqlite3 /config/manyfold.sqlite3 "UPDATE settings SET value = replace('--- \"{creator}/{modelName}/{modelId}\"\n','\n',char(10)) WHERE var == 'model_path_template';" + debug sqlite3 /config/manyfold.sqlite3 "UPDATE settings SET value = replace('--- \"{creator}/{modelName}\"\n','\n',char(10)) WHERE var == 'model_path_template';" } rename_app(){ @@ -126,26 +151,10 @@ rename_app(){ sed -i 's|powered_by_html:.*|powered_by_html: Powered by XR Forge, Manyfold, XR Fragments and NIX|g' /usr/src/app/config/locales/*.yml } -cleanup_big_files(){ - local DIR=$1 - local MAX_DAYS=$2 - local MAX_MB=$3 - test -n "$MAX_DAYS" && test -n "$MAX_MB" && { - find "$DIR" -type f -mtime +"$MAX_DAYS" -size +$(( $MAX_MB * 1024 ))k | while read file; do - echo "deleting: $file" - rm "$file" - done - } -} - -delete_big_files(){ - # check every day to delete files older than 5 days and >5mb (5mb in 2025, 6mb in 2026 e.g.) - test -z "$NO_DELETEBIGFILES" && { - everySeconds=86400 # = 1 day - maxMB=$(( $(date +%Y) - 2020 )) - maxDays=5 - $0 infinite $everySeconds cleanup_big_files /mnt/models $maxDays $maxMB & - } +start_syslog(){ + syslogd -n & # start syslogd + echocolor started syslog | logger + tail -f /var/log/messages & } # The new entrypoint of the docker @@ -153,12 +162,13 @@ boot(){ echocolor "[$APPNAME]" "booting..." test -z "$NO_OVERLAYFS" && overlayfs test -z "$NO_DEFAULTDB" && db default + start_syslog rename_app set_theme set_modelpath rclone_mount set_upload_path - delete_big_files + start_hook_daemon exec "$@" # exec prevents error 's6-overlay-suexec: fatal: can only run as pid 1' } diff --git a/manyfold/root/.config/rclone/rclone.conf b/manyfold/root/.config/rclone/rclone.conf new file mode 100644 index 0000000..d3d4319 --- /dev/null +++ b/manyfold/root/.config/rclone/rclone.conf @@ -0,0 +1,10 @@ +[scene] +type = ftp +host = ftp.scene.org +user = anonymous +pass = JMAFLA0Vk3ELCzRwhxANJZtlKai7hDW_Vw + +[modland] +type = http +url = https://modland.com + diff --git a/manyfold/root/hook.d/boot/placeholder.sh b/manyfold/root/hook.d/boot/placeholder.sh new file mode 120000 index 0000000..d264ad3 --- /dev/null +++ b/manyfold/root/hook.d/boot/placeholder.sh @@ -0,0 +1 @@ +../hourly/placeholder.sh \ No newline at end of file diff --git a/manyfold/root/hook.d/daily/delete_big_files.sh b/manyfold/root/hook.d/daily/delete_big_files.sh new file mode 100755 index 0000000..18b65a7 --- /dev/null +++ b/manyfold/root/hook.d/daily/delete_big_files.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +test -n "$NO_DELETEBIGFILES" && exit 0 # disabled + +cleanup_big_files(){ + local DIR=$1 + local MAX_DAYS=$2 + local MAX_MB=$3 + test -n "$MAX_DAYS" && test -n "$MAX_MB" && { + find "$DIR" -type f -mtime +"$MAX_DAYS" -size +$(( $MAX_MB * 1024 ))k | while read file; do + echo "deleting: $file" + rm "$file" + done + } +} + +# delete files older than 5 days and >5mb (5mb in 2025, 6mb in 2026 e.g.) +maxMB=$(( $(date +%Y) - 2020 )) +cleanup_big_files /mnt/models 5 $maxMB diff --git a/manyfold/root/hook.d/hourly/placeholder.sh b/manyfold/root/hook.d/hourly/placeholder.sh new file mode 100755 index 0000000..30d871f --- /dev/null +++ b/manyfold/root/hook.d/hourly/placeholder.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo $0 $* diff --git a/manyfold/root/hook.d/inotify_CREATE/placeholder.sh b/manyfold/root/hook.d/inotify_CREATE/placeholder.sh new file mode 120000 index 0000000..d264ad3 --- /dev/null +++ b/manyfold/root/hook.d/inotify_CREATE/placeholder.sh @@ -0,0 +1 @@ +../hourly/placeholder.sh \ No newline at end of file diff --git a/manyfold/root/hook.d/inotify_MODIFY/package_experience.sh b/manyfold/root/hook.d/inotify_MODIFY/package_experience.sh new file mode 100755 index 0000000..9dac4aa --- /dev/null +++ b/manyfold/root/hook.d/inotify_MODIFY/package_experience.sh @@ -0,0 +1,5 @@ +#!/bin/sh +echo "$1" | grep datapackage || exit 0 # nothing to do +cd $(dirname $1) +zip experience.zip *.glb +ln -s datapackage.json index.json diff --git a/manyfold/root/hook.d/inotify_MODIFY/placeholder.sh b/manyfold/root/hook.d/inotify_MODIFY/placeholder.sh new file mode 120000 index 0000000..d264ad3 --- /dev/null +++ b/manyfold/root/hook.d/inotify_MODIFY/placeholder.sh @@ -0,0 +1 @@ +../hourly/placeholder.sh \ No newline at end of file diff --git a/manyfold/root/hook.d/inotify_MOVED_TO b/manyfold/root/hook.d/inotify_MOVED_TO new file mode 120000 index 0000000..67469ce --- /dev/null +++ b/manyfold/root/hook.d/inotify_MOVED_TO @@ -0,0 +1 @@ +inotify_MODIFY \ No newline at end of file diff --git a/manyfold/xrforge.landscape.svg b/manyfold/xrforge.landscape.svg new file mode 100644 index 0000000..02dfdec --- /dev/null +++ b/manyfold/xrforge.landscape.svg @@ -0,0 +1,5 @@ + + +alpine.js + tailwind/picnic css + trystero + https://github.com/Y2Z/monolithhttps://sconstantinides.github.io/FormButtonsmetaverse name: risapassword: *****your channel name/index.glb/index.jpg/planetB.fbx/planetB.jpgrclone?index.glbplanetB.glbstream live1Fichier,Alibaba Cloud (Aliyun),Amazon Drive,Amazon S3,Backblaze B2,Box,Cache,Chunker,Crypt,Dropbox,FTP,Google Cloud Storage,Google Drive,Google Photos,Hubic,Jottacloud,Koofr,Local,Mailru,Microsoft Azure Blob Storage,Microsoft OneDrive,Mega,NAS,OpenDrive,OpenStack Swift,pCloud,QingStor,Rackspace Cloud Files,S3,SFTP,WebDAV,Yandex Disk,Webdavpaid integration-work?:paid:own instance (private hosting) my.xrchive.org subdomainuser-accounts (armadietto)XR Fragments ✓ 4 warnings xmanyfoldxrchive.orgarchive.orgmetaverse name: risapassword: *****upload/select models from RS[*] stream realtime[*] stream nonstop (paid)[*] export to HTML-file[*] export to ZIP[*] export to archive.orgxrdrive.orgrisa > index.glbstream livemanyfoldarchive.orgselfhosted via armadietto + ~/.config.jsonunhosted p2p 3D scenes via static-html-file1. load html-file / scan QR code [codeberg page]2. immersive yourself into one of the various entrypoints3. optional: connect storage 3.1. add glb 3.3 generate thumbs of hrefs4 report/scan for hrefs to other world (goto 1)5. export new QR + public linklocal diskfrom URLlocal diskfrom URLXR Fragments ✓ 4 warnings xxrdrive.org/myorg/risaxrfsignup loginplanetC.glbplanetB.glbplanetC.glblikestore likes as xttr on glb-file, cache as index.jsonmobile UI: https://codepen.io/san2509/pen/NWMPVzbsearch herepowered by xrfragments, webxrcast, remotestorage.jsxrshrisa $ ls planetB planetC$ cd planetBxrsh$ ls planetB planetC$ cd planetBgit repo with offline PWA serviceworkerbundle as single-file-html PWAcss styling/theme/notifications/uiadd QR-code generatorp2p QRadd remotestorage dialogremotestorage projectfolder-pickerstyle thumbnail-windowsmake responsiveadd glb filetype extensionadd xrsh (offline-first)export: to remotestorage folder(picker)export: as single-file htmlexport: as zip-file1. load html-file / scan QR code [codeberg page]2. immersive yourself into one of the various entrypoints3. optional: connect storage3.1. add glb3.2 (re)generate thumbs of hrefs4 report/scan for hrefs to other world (goto 1)5. export new QR + public linkdocumentation git-repouserstoryexport appexport appxrdrive.orgsignup loginplanetB.glbplanetC.glbsearch herepowered by xrfragments, webxrcast, remotestorage.jsplanetB.glbplanetC.glbplanetB.glbplanetC.glbplanetB.glbplanetC.glbmetaverse name: risapassword: *****staff picks:free:3 channels (max 10MB) --> more = selfhost or pay[v] submit to xrdrive homepageoffline remotestorage PWASaaS wrapperblender pluginimportmanyfoldxrchive.orgarchive.orglocal diskfrom URLS3 metadata queriessaverisa > index.glbstream livemanyfoldarchive.orglocal diskfrom URLXR Fragments ✓ 4 warnings xsignup loginplanetC.glblikexrshrisa $ ls planetB planetC$ cd planetBexport appmetaverse name: risapassword: *****[v] submit to xrdrive homepage5apps.comgit repo with nix filepatch/json-config: add menu with default [iframe] app(s)patch/json-config: add xrdrive.html + set as default iframe apppatch/json-config: template variable overrides from JSON (render monkeypatch)documentation git-reposponsor (xrdrive.org)sponsor(xrdrive.org)sponsor (xrdrive.org)sponsor (xrdrive.org)own xrdrive.jsonRS app: xr overview (+staff picks)eed.htmlxrdrive.nixcopy to storageplanetB.glbcopy to storagexrdrive.org \ No newline at end of file diff --git a/nix/docker.nix b/nix/docker.nix index 84d564d..39c7b09 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -36,6 +36,9 @@ rec pkgs.pkgsStatic.sqlite pkgs.pkgsStatic.rclone pkgs.pkgsStatic.fuse3 + pkgs.pkgsStatic.acl # getfacl e.g. + pkgs.pkgsStatic.inotify-tools # inotifywait e.g. + pkgs.pkgsStatic.zip # inotifywait e.g. ./.. ]; };