optimized rclone http server

manyfold/README.md

@@ -13,15 +13,21 @@ $ docker load < $(nix-build nix/docker.nix)
 
 # Extra environment-variables
 
-| environment variable | default      | info                   |
-|----------------------|--------------|------------------------|
-| `APPNAME`            | `manyfold`   | manyfold instance name |
-| `THEME`              | `default`    | bootstrap theme        |
-| `NO_OVERLAYFS`       | ``           | disable the filesystem overlay mechanism |
-| `NO_DEFAULTDB`       | ``           | disable the default db (activates manyfold installer) |
-| `NO_DELETEBIGFILES`  | ``           | disable deleting big files which are older than 5 days and bigger than ($currentyear-2020) MB's |
-| `RCLONE_REMOTE`      | ``           | specify **single** rclone remote name (without semicolon) to mount |
-| `UPLOAD_PATH`        | `/mnt/models`| specify default library where user-files are uploaded (regular dir or mounted rclone path) |
+| environment variable  | default      | info                   |
+|-----------------------|--------------|------------------------|
+| `APPNAME`             | `manyfold`   | manyfold instance name |
+| `THEME`               | `default`    | bootstrap theme        |
+| `NO_OVERLAYFS`        | ``           | disable the filesystem overlay mechanism |
+| `NO_DEFAULTDB`        | ``           | disable the default db (activates manyfold installer) |
+| `NO_DELETEBIGFILES`   | ``           | disable deleting big files which are older than 5 days and bigger than ($currentyear-2020) MB's |
+| `NO_PACKAGEALL`       | ``           | don't package all experiences every hour to /usr/src/app/public/experiences.zip |
+| `RCLONE_REMOTE`       | ``           | specify **single** rclone remote name (without semicolon) to mount (default: mount all rclone remotes)|
+| `UPLOAD_PATH`         | `/mnt/models`| specify default library where user-files are uploaded (regular dir or mounted rclone path) |
+| `FEDERATE_DRIVE_PATH` | `/mnt/models`| serve path over HTTP (so other instances can add it as a remote). Specify `0` to disable |
+| `FEDERATE_DRIVE_PORT` | `3215`       | specify default library where user-files are uploaded (regular dir or mounted rclone path) |
+| `FEDERATE_DRIVE_USER` | ``           | specify HTTP AUTH credentials (`user` e.g.) for restricted sharing |          
+| `FEDERATE_DRIVE_PW`   | ``           | specify HTTP AUTH credentials (`pass` e.g.) for restricted sharing |          
+| `FEDERATE_DRIVE_CACHE`| `1m0s`       | specify interval to re-check all models/directories |
 
 # Default database / admin login
 
@@ -36,7 +42,7 @@ $ docker load < $(nix-build nix/docker.nix)
 The server-image will boot `manyfold/cli/manyfold.sh boot` and check for directory `/manyfold` (in the container).
 When found, it uses the files in there instead (`/manyfold/usr/src/app/public/404.html` instead of `/usr/src/app/public/404.html` e.g.).
 
-# Remote network-drives (with models)
+# Federated drives (inbound)
 
 > Thanks to [rclone](https://rclone.org) network-drives automatically show up as manyfold libraries.
 
@@ -55,6 +61,71 @@ Your drives will get automagically mounted and added to the database automagical
 
 TIP: use **alphanumeric** names for rclone remotes (manyfold libraries choke on dot- or other special-characters)
 
+
+By default environment-flag `FEDERATE_DRIVE_PATH` will share path `/mnt/models` as an open web directory.
+This means it can be added as remote by other instances.
+See the environment-flags for more options.
+
+<details>
+    <summary>**Example connect to other XRForge instance**</summary>
+<br>
+```
+$ rclone config                                                                                                                                                                                                                                                           
+Current remotes:                                                                                                                                                                                                                                                                          
+                                                                                                                                                                                                                                                                                          
+Name                 Type                                                                                                                                                                                                                                                                 
+====                 ====                                                                                                                                                                                                                                                                 
+                                                                                                                                                                                                                                                                                          
+e) Edit existing remote                                                                                                                                                                                                                                                                   
+n) New remote                                                                                                                                                                                                                                                                             
+d) Delete remote                                                                                                                                                                                                                                                                          
+r) Rename remote                                                      
+c) Copy remote                                                        
+s) Set configuration password                                         
+q) Quit config                                                        
+e/n/d/r/c/s/q> n                                                      
+
+Enter name for new remote.                                            
+name> xrforge_instanceC                                                            
+
+Option Storage.                                                       
+Type of storage to configure.                                         
+Choose a number from below, or type in your own value.
+
+...
+22 / HTTP
+...
+
+Storage> 22
+
+Option url.
+URL of HTTP host to connect to.
+E.g. "https://example.com", or "https://user:pass@example.com" to use a username and password.
+Enter a value.
+url> http://url-to-another-xrforge-instance.com
+
+Option no_escape.
+Do not escape URL metacharacters in path names.
+Enter a boolean value (true or false). Press Enter for the default (false).
+no_escape> 
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+                                   
+Configuration complete.      
+Options:             
+- type: http                      
+- url: http://localhost:8791                                          
+Keep this "test" remote?                                              
+y) Yes this is OK (default)        
+e) Edit this remote      
+d) Delete this remote    
+y/e/d> y
+```
+</details>
+
 # Unixy event hooks 
 
 Until WebEvents [will get implemented on a REST-level in manyfold](https://github.com/orgs/manyfold3d/projects/4/views/1?filterQuery=Pub&pane=issue&itemId=108834509&issue=manyfold3d%7Cmanyfold%7C4097) Things like boot-phase, scheduler and file-changes can be reacted up via the `/root/hook.d` directory:
@@ -72,6 +143,8 @@ You can put scripts in there, which are fired when needed.
 
 > Example: [manyfold/root/hook.d/daily/delete_big_files.sh] is triggered daily to cleanup files which exceed a certain age/size.
 
-Currently inotify events (`inotify_MODIFY` e.g.) are triggered when files in `/mnt/models` change.
+Currently inotify events (`inotify_MODIFY` e.g.) are triggered for local file-changes (`/mnt/models` e.g.).
+In theory, federated drives can still be reacted upon, but by integrating with XRForge's ActivityPub (**Follow** feature e.g.)
 
 > Perhaps in the future this will also work for rclone remotes, by writing a `hourly`-script which scans them and fires `inotify_MODIFY` accordingly.
+

manyfold/cli/manyfold.sh

@@ -22,13 +22,14 @@ run(){
       #-e NO_DEFAULTDB=true \
       #-e PUBLIC_HOSTNAME=localhost \
       #-e PUBLIC_PORT=80 \
-  debug ${oci} run "$@" -p 8790:3214 --name xrforge \
+  debug ${oci} run "$@" -p 8790:3214 -p 8791:3215 --name xrforge \
       -e SECRET_KEY_BASE=lkjwljlkwejrlkjek34k234l \
       -e DATABASE_ADAPTER=sqlite3 \
       -e SUDO_RUN_UNSAFELY=enabled \
       -e MULTIUSER=enabled \
       -e FEDERATION=enabled \
       -e THEME=vapor \
+      -e FEDERATE_DRIVE_CACHE=5s \
       --cap-add SYS_ADMIN --security-opt apparmor:unconfined --device /dev/fuse \
       xrforge 
       #ghcr.io/manyfold3d/manyfold-solo:latest
@@ -77,7 +78,7 @@ start_hook_daemon(){
   $0 infinite 86400 hook daily &
   $0 infinite 3600  hook hourly &
   # trigger hooks when files change in /mnt/models
-  inotifywait -r -m /mnt/models | awk '$2 ~ /(CREATE|MODIFY|MOVED_TO)/ { system("'$0' hook inotify_"$2" "$1""$3) }' &
+  inotifywait -r -m /mnt/models | awk '$2 ~ /(CREATE|MODIFY|MOVED_TO|DELETE)/ { system("'$0' hook inotify_"$2" "$1""$3) }' &
 }
 
 
@@ -148,7 +149,10 @@ set_modelpath(){
 rename_app(){
   echocolor "[$APPNAME]" "renaming manyfold to $APPNAME"
   sed -i 's/title: Manyfold/title: '$APPNAME'/g' /usr/src/app/config/locales/*.yml
-  sed -i 's|powered_by_html:.*|powered_by_html: Powered by <a href="https://forgejo.isvery.ninja/coderofsalvation/xrforge">XR Forge</a>, <a href="https://manifold.app" target="_blank">Manyfold</a>, <a href="https://xrfragment.org">XR Fragments</a> and <a href="https://nixos.org" target="_blank">NIX</a>|g' /usr/src/app/config/locales/*.yml
+  sed -i 's|powered_by_html:.*|powered_by_html: Radically opensource-powered by <a href="https://forgejo.isvery.ninja/coderofsalvation/xrforge">XR Forge</a>, <a href="https://manifold.app" target="_blank">Manyfold</a>, <a href="https://xrfragment.org">XR Fragments</a> and <a href="https://nixos.org" target="_blank">NIX</a>|g' /usr/src/app/config/locales/*.yml
+
+  sed -i 's|Models|Experiences|g' /usr/src/app/config/locales/*.yml
+  sed -i 's|Model|Experience|g' /usr/src/app/config/locales/*.yml
 }
 
 start_syslog(){
@@ -169,6 +173,7 @@ boot(){
   rclone_mount
   set_upload_path
   start_hook_daemon
+  hook boot # emit unixy hook-event (/root/hook.d/boot/* scripts)
 
   exec "$@" # exec prevents error 's6-overlay-suexec: fatal: can only run as pid 1'
 }

manyfold/root/hook.d/boot/httpserver.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+test -z "$FEDERATE_DRIVE_PATH" && FEDERATE_DRIVE_PATH=/mnt/models
+test -z "$FEDERATE_DRIVE_PORT" && FEDERATE_DRIVE_PORT=3215
+test -z "$FEDERATE_DRIVE_CACHE" && FEDERATE_DRIVE_CACHE=1m0s
+
+test "$FEDERATE_DRIVE_PATH" = 0 && exit 0 # nothing to do (disabled)
+
+test -n "$FEDERATE_DRIVE_USER" && test -m "$FEDERATE_DRIVE_PW" && {
+  AUTH="--user $FEDERATE_DRIVE_USER --pass $FEDERATE_DRIVE_PW"
+}
+
+set -x
+rclone serve http \
+  --poll-interval $FEDERATE_DRIVE_CACHE \
+  --addr 0.0.0.0:$FEDERATE_DRIVE_PORT ${AUTH} $FEDERATE_DRIVE_PATH &> /var/log/rclone.log &

manyfold/root/hook.d/daily/package_experiences_zip.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+test -n "$NO_PACKAGEALL" && exit 0; # nothing to do
+cd /mnt/models 
+{
+  echo "# XRForge experiences"
+  echo ""
+  echo "XRForge [$PUBLIC_HOSTNAME] snapshot made at $(date)"
+  echo ""
+  echo "# How to view"
+  echo ""
+  echo "You'll need a 3D app/website/viewer with:"
+  echo ""
+  echo "* support for multiple 3D fileformats"
+  echo "* XR Fragments support (at least href extras) to teleport between them"
+} > /mnt/models/README.md
+
+zip -r /usr/src/app/public/experiences.zip *

manyfold/root/hook.d/hourly/placeholder.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-echo $0 $*

manyfold/root/hook.d/inotify_MODIFY/placeholder.sh

@@ -1 +0,0 @@
-../hourly/placeholder.sh

nix/docker.nix

@@ -2,7 +2,21 @@
   , pkgsLinux ? import <nixpkgs> { system = "x86_64-linux"; } 
 }:
 
-let 
+let
+
+# -- swap the above pkgs with the pkgs definition below for 100% reproducability
+# -- WARNING: requires quite some diskspace 
+
+#{ } :
+#
+#let 
+#  pkgs = import ( builtins.fetchTarball {
+#    # https://github.com/NixOS/nixpkgs/archive/refs/heads/nixos-25.05.zip
+#    name = "nixos-25.05";
+#    url  = "https://github.com/NixOS/nixpkgs/archive/nixos-25.05.tar.gz";
+#    sha256 = "sha256:10cqhkqkifcgyibj9nwxrnq424crfl40kwr3daky83m2fisb4f6p";
+#  }) {};
+
   vars = import ./vars.nix;
 
   # generate the reproducable blob below via: