milestone 4g: manyfold.sh auto-generate experience.zip index.json on change (inotify)

manyfold/README.md

@@ -19,6 +19,7 @@ $ docker load < $(nix-build nix/docker.nix)
 | `THEME`              | `default`    | bootstrap theme        |
 | `NO_OVERLAYFS`       | ``           | disable the filesystem overlay mechanism |
 | `NO_DEFAULTDB`       | ``           | disable the default db (activates manyfold installer) |
+| `NO_DELETEBIGFILES`  | ``           | disable deleting big files which are older than 5 days and bigger than ($currentyear-2020) MB's |
 | `RCLONE_REMOTE`      | ``           | specify **single** rclone remote name (without semicolon) to mount |
 | `UPLOAD_PATH`        | `/mnt/models`| specify default library where user-files are uploaded (regular dir or mounted rclone path) |
 
@@ -54,3 +55,23 @@ Your drives will get automagically mounted and added to the database automagical
 
 TIP: use **alphanumeric** names for rclone remotes (manyfold libraries choke on dot- or other special-characters)
 
+# Unixy event hooks 
+
+Until WebEvents [will get implemented on a REST-level in manyfold](https://github.com/orgs/manyfold3d/projects/4/views/1?filterQuery=Pub&pane=issue&itemId=108834509&issue=manyfold3d%7Cmanyfold%7C4097) Things like boot-phase, scheduler and file-changes can be reacted up via the `/root/hook.d` directory:
+
+```
+$ ls /root/hook.d
+boot
+daily
+hourly
+inotify_CREATE
+inotify_MODIFY
+```
+
+You can put scripts in there, which are fired when needed.
+
+> Example: [manyfold/root/hook.d/daily/delete_big_files.sh] is triggered daily to cleanup files which exceed a certain age/size.
+
+Currently inotify events (`inotify_MODIFY` e.g.) are triggered when files in `/mnt/models` change.
+
+> Perhaps in the future this will also work for rclone remotes, by writing a `hourly`-script which scans them and fires `inotify_MODIFY` accordingly.

manyfold/cli/manyfold.sh

@@ -56,6 +56,31 @@ infinite(){
   loop "$@"
 }
 
+# flexible unixy hook-mechanism which executes all files in ~/hook.d/foo/* when 
+# calling 'hook foo bar'
+hook(){
+  test -z "$1" && { echo "usage: hook <cmd_or_jsfunction> [args]"; return 0; } 
+  logger "$ hook $*"
+  cmd=$1
+  shift 
+  test -d ~/hook.d/$cmd && {
+    find -L ~/hook.d/$cmd/ -type f -executable -maxdepth 1 | while read hook; do 
+      logger "    |+ hook $hook $*"
+      { $hook "$@" || true; } 2>&1 |  awk '{ gsub(/\/root\/\//,"",$1); $1 = sprintf("%-40s", $1)} 1' | logger
+    done
+  }
+}
+
+start_hook_daemon(){
+  # every day call scripts in ~/hook.d/cleanup_daily/*
+  # 86400 secs = 1 day    3600 = 1 hour
+  $0 infinite 86400 hook daily &
+  $0 infinite 3600  hook hourly &
+  # trigger hooks when files change in /mnt/models
+  inotifywait -r -m /mnt/models | awk '$2 ~ /(CREATE|MODIFY|MOVED_TO)/ { system("'$0' hook inotify_"$2" "$1""$3) }' &
+}
+
+
 db(){
   default(){
     test -f /manyfold/manyfold.sql && ! test -f /config/manyfold.sql && {
@@ -117,7 +142,7 @@ set_theme(){
 
 set_modelpath(){
   echocolor "[$APPNAME]" "enforcing modelpath"
-  debug sqlite3 /config/manyfold.sqlite3 "UPDATE settings SET value = replace('--- \"{creator}/{modelName}/{modelId}\"\n','\n',char(10)) WHERE var == 'model_path_template';"
+  debug sqlite3 /config/manyfold.sqlite3 "UPDATE settings SET value = replace('--- \"{creator}/{modelName}\"\n','\n',char(10)) WHERE var == 'model_path_template';"
 }
 
 rename_app(){
@@ -126,26 +151,10 @@ rename_app(){
   sed -i 's|powered_by_html:.*|powered_by_html: Powered by <a href="https://forgejo.isvery.ninja/coderofsalvation/xrforge">XR Forge</a>, <a href="https://manifold.app" target="_blank">Manyfold</a>, <a href="https://xrfragment.org">XR Fragments</a> and <a href="https://nixos.org" target="_blank">NIX</a>|g' /usr/src/app/config/locales/*.yml
 }
 
-cleanup_big_files(){
+start_syslog(){
-  local DIR=$1
+  syslogd -n & # start syslogd
-  local MAX_DAYS=$2
+  echocolor started syslog | logger 
-  local MAX_MB=$3
+  tail -f /var/log/messages &
-  test -n "$MAX_DAYS" && test -n "$MAX_MB" && {
-    find "$DIR" -type f -mtime +"$MAX_DAYS" -size +$(( $MAX_MB * 1024 ))k | while read file; do 
-      echo "deleting: $file"
-      rm "$file"
-    done 
-  }
-}
-
-delete_big_files(){
-  # check every day to delete files older than 5 days and >5mb (5mb in 2025, 6mb in 2026 e.g.)
-  test -z "$NO_DELETEBIGFILES" && {
-    everySeconds=86400 # = 1 day
-    maxMB=$(( $(date +%Y) - 2020 ))
-    maxDays=5
-    $0 infinite $everySeconds cleanup_big_files /mnt/models $maxDays $maxMB & 
-  }
 }
 
 # The new entrypoint of the docker 
@@ -153,12 +162,13 @@ boot(){
   echocolor "[$APPNAME]" "booting..."
   test -z "$NO_OVERLAYFS" && overlayfs
   test -z "$NO_DEFAULTDB" && db default
+  start_syslog 
   rename_app
   set_theme
   set_modelpath
   rclone_mount
   set_upload_path
-  delete_big_files
+  start_hook_daemon
 
   exec "$@" # exec prevents error 's6-overlay-suexec: fatal: can only run as pid 1'
 }

manyfold/root/.config/rclone/rclone.conf

@@ -0,0 +1,10 @@
+[scene]
+type = ftp
+host = ftp.scene.org
+user = anonymous
+pass = JMAFLA0Vk3ELCzRwhxANJZtlKai7hDW_Vw
+
+[modland]
+type = http
+url = https://modland.com
+

manyfold/root/hook.d/boot/placeholder.sh

@@ -0,0 +1 @@
+../hourly/placeholder.sh

manyfold/root/hook.d/daily/delete_big_files.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+  
+test -n "$NO_DELETEBIGFILES" && exit 0 # disabled
+
+cleanup_big_files(){
+  local DIR=$1
+  local MAX_DAYS=$2
+  local MAX_MB=$3
+  test -n "$MAX_DAYS" && test -n "$MAX_MB" && {
+    find "$DIR" -type f -mtime +"$MAX_DAYS" -size +$(( $MAX_MB * 1024 ))k | while read file; do 
+      echo "deleting: $file"
+      rm "$file"
+    done 
+  }
+}
+
+# delete files older than 5 days and >5mb (5mb in 2025, 6mb in 2026 e.g.)
+maxMB=$(( $(date +%Y) - 2020 ))
+cleanup_big_files /mnt/models 5 $maxMB 

manyfold/root/hook.d/hourly/placeholder.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo $0 $*

manyfold/root/hook.d/inotify_CREATE/placeholder.sh

@@ -0,0 +1 @@
+../hourly/placeholder.sh

manyfold/root/hook.d/inotify_MODIFY/package_experience.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+echo "$1" | grep datapackage || exit 0 # nothing to do
+cd $(dirname $1)
+zip experience.zip *.glb 
+ln -s datapackage.json index.json

manyfold/root/hook.d/inotify_MODIFY/placeholder.sh

@@ -0,0 +1 @@
+../hourly/placeholder.sh

manyfold/root/hook.d/inotify_MOVED_TO

@@ -0,0 +1 @@
+inotify_MODIFY

nix/docker.nix

@@ -36,6 +36,9 @@ rec
         pkgs.pkgsStatic.sqlite
         pkgs.pkgsStatic.rclone
         pkgs.pkgsStatic.fuse3
+        pkgs.pkgsStatic.acl # getfacl e.g.
+        pkgs.pkgsStatic.inotify-tools # inotifywait e.g.
+        pkgs.pkgsStatic.zip # inotifywait e.g.
         ./..
       ];
     };