added mastodon share

src/.init.lua

@@ -14,7 +14,7 @@ HidePath('/usr/share/ssl/')
 
 -- create
 app = require("soakbean") {
-  cache = { items = {}, url = {} },
+  cache = { items = {}, url = {}, shared = require('sharedcache') },
   plugin = {},
   opts  = {},
   forkpids = {},
@@ -52,7 +52,6 @@ app = require("soakbean") {
   end
 }
 
-util.fetchInstall(app) -- setup internal fetch callback mechanism 
 
 -- api skeleton  (implemented via soakbean's app.on(...))
 app.set  = function(k,v) end 
@@ -61,6 +60,8 @@ app.init = function()    end
 -- load config
 app.opts = require('config')(app)
 
+util.fetchInstall(app) -- setup internal fetch callback mechanism 
+
 -- load system plugins (in specific order for admin panel)
 app.plugin.global        = app.addPlugin('plugin/global')
 app.plugin.pubnix        = app.addPlugin('plugin/pubnix')
@@ -78,8 +79,11 @@ app.plugin.peertube      = app.addPlugin('plugin/peertube')
 app.plugin.git           = app.addPlugin('plugin/git')
 app.plugin.webhook       = app.addPlugin('plugin/webhook')
 app.plugin.archive       = app.addPlugin('plugin/archive')
+--app.plugin.scheduler     = app.addPlugin('plugin/scheduler')    -- should always be second last
 app.plugin.staticwebgen  = app.addPlugin('plugin/staticwebgen') -- should always be last
 
+if unix.getpid() == 0 then unix.exit(0) end -- child-threads shall not pass beyond here
+
 app.runcmd(app) -- run clicmds
 
 print = function(...) Log(kLogInfo, ... or "''") end -- nice redbean logging from now on

src/.lua/sharedcache.lua

@@ -0,0 +1,29 @@
+local sqlite3   = require 'lsqlite3'
+local sharedcache  =  { db= sqlite3.open( arg[-1] .. ".sqlite3" ) }
+
+-- sqlite3 will be used for IPC data
+sharedcache.update = function(t)
+  print_r(t)
+  local stmt = sharedcache.db:prepare("UPDATE sharedcache SET json = ?;")
+  local jsonstr, err = EncodeJson(t)
+  if err then 
+    print_r(t)
+    return print("sharecache.update(): " .. err) 
+  end
+  stmt:bind_values( jsonstr )
+  stmt:step()
+  stmt:finalize()
+end
+
+sharedcache.get = function()
+  for row in sharedcache.db:nrows("SELECT * FROM sharedcache;") do
+    print_r(row)
+    return DecodeJson( row.json )
+  end
+  return {empty=true}
+end
+sharedcache.db:exec('CREATE TABLE sharedcache ( id INTEGER PRIMARY KEY, json TEXT);')
+sharedcache.db:exec("INSERT INTO sharedcache (id,json) VALUES(1, NULL);")
+sharedcache.update({}) -- weird: for some reason the NULL + this update is needed for proper json
+
+return sharedcache

src/.lua/util.lua

@@ -310,21 +310,27 @@ function util.pluck(path, tbl, default)
 end
 
 function util.fetchInstall(app)
-  app.expectedreq = {}
+  app.cache.expectedreq = {}
   app.post('/bus',function(req,res,next)
     local reqid = GetParam('id')
-    local cb = app.expectedreq[reqid]
+    local cb = app.cache.expectedreq[reqid]
-    if cb then cb(res,req,next) end -- important: flipped req,res
+    if cb then 
+      cb(res,req,next) -- important: flipped req,res
+      app.cache.expectedreq[reqid] = nil
+    else
+      trace("util.fetch: reqid not found " .. reqid .. " pid=" .. unix.getpid())
+    end
   end)
 end
 
 function util.fetchServer(url, opts, cb)
-  util.fetch( util.getURL(url,{server=true}), nil, cb )
+  util.fetch( util.getURL(url, {server=true}), opts, cb )
 end
 
 function util.fetch(url, opts, cb)
-  local reqid = tostring( Rdrand() )
+  local reqid       = tostring( math.abs(Rdrand()) )
-  app.expectedreq[reqid] = cb
+  print("reqid = " .. reqid .. " pid:" .. unix.getpid() )
+  app.cache.expectedreq[reqid] = cb
   local urlret = util.getURL('/bus?id=' .. reqid, {server=true} ) 
   if assert(unix.fork()) == 0 then
     print('🤞 fetch ' .. url)
@@ -367,4 +373,42 @@ util.queueServerRequest = function(method, path, params, host, headers, body, sc
   Route      = function() end
 end
 
+function util.traverse(arr, cb, key, parent)
+  if key == nil then key = 'children' end
+  foreach( arr, function(k,child) 
+    if type(child) == 'table' then 
+      cb(child,parent)
+      if type(child[key]) == 'table' then
+        if child[key][1] then
+          util.traverse( child[key], cb, key, child )
+        end
+      end
+    end
+  end)
+end
+
+function util.traverseXML( parsedXml, cb)
+  util.traverse( parsedXml, 
+    function(node, parent)
+      local struct = { tag = '?', prop = {} }
+      if node['name'] ~= nil then struct['tag'] = node:name():lower() end
+      foreach( node['___props'], function(k,v) -- move to props array
+        local val = node[ '@' .. v['name'] ]
+        if type(val) == "table" then 
+          foreach( val, function(k,v) val = v end) -- pick last
+        end
+        struct['prop'][ v['name'] ] = val 
+      end)
+      if parent ~= nil then 
+        struct['parent'] = (function(parent) 
+          return function() return parent end 
+        end)(parent)
+      end
+      cb(struct,node)
+    end,
+    '___children'
+  )
+end
+
+
 return util

src/.lua/xmlSimple.lua

@@ -0,0 +1,174 @@
+---------------------------------------------------------------------------------
+---------------------------------------------------------------------------------
+--@LIB
+-- xml.lua - XML parser for use with the Corona SDK.
+-- https://github.com/Cluain/Lua-Simple-XML-Parser
+--
+-- version: 1.2
+--
+-- CHANGELOG:
+--
+-- 1.2 - Created new structure for returned table
+-- 1.1 - Fixed base directory issue with the loadFile() function.
+--
+-- NOTE: This is a modified version of Alexander Makeev's Lua-only XML parser
+-- found here: http://lua-users.org/wiki/LuaXml
+--
+---------------------------------------------------------------------------------
+---------------------------------------------------------------------------------
+function newParser()
+
+    XmlParser = {};
+
+    function XmlParser:ToXmlString(value)
+        value = string.gsub(value, "&", "&"); -- '&' -> "&"
+        value = string.gsub(value, "<", "&lt;"); -- '<' -> "&lt;"
+        value = string.gsub(value, ">", "&gt;"); -- '>' -> "&gt;"
+        value = string.gsub(value, "\"", "&quot;"); -- '"' -> "&quot;"
+        value = string.gsub(value, "([^%w%&%;%p%\t% ])",
+            function(c)
+                return string.format("&#x%X;", string.byte(c))
+            end);
+        return value;
+    end
+
+    function XmlParser:FromXmlString(value)
+        value = string.gsub(value, "&#x([%x]+)%;",
+            function(h)
+                return string.char(tonumber(h, 16))
+            end);
+        value = string.gsub(value, "&#([0-9]+)%;",
+            function(h)
+                return string.char(tonumber(h, 10))
+            end);
+        value = string.gsub(value, "&quot;", "\"");
+        value = string.gsub(value, "&apos;", "'");
+        value = string.gsub(value, "&gt;", ">");
+        value = string.gsub(value, "&lt;", "<");
+        value = string.gsub(value, "&amp;", "&");
+        return value;
+    end
+
+    function XmlParser:ParseArgs(node, s)
+        string.gsub(s, "(%w+)=([\"'])(.-)%2", function(w, _, a)
+            node:addProperty(w, self:FromXmlString(a))
+        end)
+    end
+
+    function XmlParser:ParseXmlText(xmlText)
+        local stack = {}
+        local top = newNode()
+        table.insert(stack, top)
+        local ni, c, label, xarg, empty
+        local i, j = 1, 1
+        while true do
+            ni, j, c, label, xarg, empty = string.find(xmlText, "<(%/?)([%w_:]+)(.-)(%/?)>", i)
+            if not ni then break end
+            local text = string.sub(xmlText, i, ni - 1);
+            if not string.find(text, "^%s*$") then
+                local lVal = (top:value() or "") .. self:FromXmlString(text)
+                stack[#stack]:setValue(lVal)
+            end
+            if empty == "/" then -- empty element tag
+                local lNode = newNode(label)
+                self:ParseArgs(lNode, xarg)
+                top:addChild(lNode)
+            elseif c == "" then -- start tag
+                local lNode = newNode(label)
+                self:ParseArgs(lNode, xarg)
+                table.insert(stack, lNode)
+		top = lNode
+            else -- end tag
+                local toclose = table.remove(stack) -- remove top
+
+                top = stack[#stack]
+                if #stack < 1 then
+                    error("XmlParser: nothing to close with " .. label)
+                end
+                if toclose:name() ~= label then
+                    error("XmlParser: trying to close " .. toclose.name .. " with " .. label)
+                end
+                top:addChild(toclose)
+            end
+            i = j + 1
+        end
+        local text = string.sub(xmlText, i);
+        if #stack > 1 then
+            error("XmlParser: unclosed " .. stack[#stack]:name())
+        end
+        return top
+    end
+
+    function XmlParser:loadFile(xmlFilename, base)
+        if not base then
+            base = system.ResourceDirectory
+        end
+
+        local path = system.pathForFile(xmlFilename, base)
+        local hFile, err = io.open(path, "r");
+
+        if hFile and not err then
+            local xmlText = hFile:read("*a"); -- read file content
+            io.close(hFile);
+            return self:ParseXmlText(xmlText), nil;
+        else
+            print(err)
+            return nil
+        end
+    end
+
+    return XmlParser
+end
+
+function newNode(name)
+    local node = {}
+    node.___value = nil
+    node.___name = name
+    node.___children = {}
+    node.___props = {}
+
+    function node:value() return self.___value end
+    function node:setValue(val) self.___value = val end
+    function node:name() return self.___name end
+    function node:setName(name) self.___name = name end
+    function node:children() return self.___children end
+    function node:numChildren() return #self.___children end
+    function node:addChild(child)
+        if self[child:name()] ~= nil then
+            if type(self[child:name()].name) == "function" then
+                local tempTable = {}
+                table.insert(tempTable, self[child:name()])
+                self[child:name()] = tempTable
+            end
+            table.insert(self[child:name()], child)
+        else
+            self[child:name()] = child
+        end
+        table.insert(self.___children, child)
+    end
+
+    function node:properties() return self.___props end
+    function node:numProperties() return #self.___props end
+    function node:addProperty(name, value)
+        local lName = "@" .. name
+        if self[lName] ~= nil then
+            if type(self[lName]) == "string" then
+                local tempTable = {}
+                table.insert(tempTable, self[lName])
+                self[lName] = tempTable
+            end
+            table.insert(self[lName], value)
+        else
+            self[lName] = value
+        end
+        table.insert(self.___props, { name = name, value = self[name] })
+    end
+
+    return node
+end
+
+
+return {
+  newNode = newNode,
+  newParser = newParser
+}

src/css/xrforge.css

@@ -182,6 +182,13 @@ h1{
   font-size: 29px;
   margin: 15px 0px;
 }
+
+h1 a,
+h2 a,
+center a{
+  text-decoration:none;
+  border:none !important;
+}
 .sym{ font-size:100px; color:#000; }
 
 

src/css/xrforge.max.css

@@ -32,11 +32,15 @@ body{
 
 @media (max-width: 470px){
   .menu a:nth-child(2){
-    display:none;
+    display:  text-decoration:none;
+    border:none;none;
   }
 }
 
 @media (max-width: 890px){
+  table.footer tr td {
+    display: table-row;
+  }
   .grid .tile {
     width:47%;
   }
@@ -62,6 +66,14 @@ body{
   font-size:14px;
 }
 
+img.icon.rss{
+  width: 14px;
+  transform: translate(0,-4px);
+}
+img.icon.mastodon{
+  width: 19px;
+  transform: translate(-4px,1px);
+}
 img.icon.invert,
 img.icon.link,
 img.icon.play,

src/img/mastodon.svg

@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="61.076954mm" height="65.47831mm" viewBox="0 0 216.4144 232.00976">
+  <path fill="#8888aa" d="M211.80734 139.0875c-3.18125 16.36625-28.4925 34.2775-57.5625 37.74875-15.15875 1.80875-30.08375 3.47125-45.99875 2.74125-26.0275-1.1925-46.565-6.2125-46.565-6.2125 0 2.53375.15625 4.94625.46875 7.2025 3.38375 25.68625 25.47 27.225 46.39125 27.9425 21.11625.7225 39.91875-5.20625 39.91875-5.20625l.8675 19.09s-14.77 7.93125-41.08125 9.39c-14.50875.7975-32.52375-.365-53.50625-5.91875C9.23234 213.82 1.40609 165.31125.20859 116.09125c-.365-14.61375-.14-28.39375-.14-39.91875 0-50.33 32.97625-65.0825 32.97625-65.0825C49.67234 3.45375 78.20359.2425 107.86484 0h.72875c29.66125.2425 58.21125 3.45375 74.8375 11.09 0 0 32.975 14.7525 32.975 65.0825 0 0 .41375 37.13375-4.59875 62.915"/>
+  <path fill="#000000" d="M177.50984 80.077v60.94125h-24.14375v-59.15c0-12.46875-5.24625-18.7975-15.74-18.7975-11.6025 0-17.4175 7.5075-17.4175 22.3525v32.37625H96.20734V85.42325c0-14.845-5.81625-22.3525-17.41875-22.3525-10.49375 0-15.74 6.32875-15.74 18.7975v59.15H38.90484V80.077c0-12.455 3.17125-22.3525 9.54125-29.675 6.56875-7.3225 15.17125-11.07625 25.85-11.07625 12.355 0 21.71125 4.74875 27.8975 14.2475l6.01375 10.08125 6.015-10.08125c6.185-9.49875 15.54125-14.2475 27.8975-14.2475 10.6775 0 19.28 3.75375 25.85 11.07625 6.36875 7.3225 9.54 17.22 9.54 29.675"/>
+</svg>

src/img/rss.svg

@@ -0,0 +1,4 @@
+<svg fill="#8888aa" role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
+<title>RSS</title>
+<path d="M19.199 24C19.199 13.467 10.533 4.8 0 4.8V0c13.165 0 24 10.835 24 24h-4.801zM3.291 17.415c1.814 0 3.293 1.479 3.293 3.295 0 1.813-1.485 3.29-3.301 3.29C1.47 24 0 22.526 0 20.71s1.475-3.294 3.291-3.295zM15.909 24h-4.665c0-6.169-5.075-11.245-11.244-11.245V8.09c8.727 0 15.909 7.184 15.909 15.91z"/>
+</svg>

src/js/scraper.js

@@ -0,0 +1,7 @@
+function Scraper(domsel){
+  let el = document.querySelector(domsel)
+  if( !el ) throw 'scraper.js: invalid domselector'
+  alert("oi")
+}
+
+new Scraper('#scraper')

src/page/dir.html

@@ -3,9 +3,19 @@ ${header}
 {if parent ~= nil and parent.parent == nil then
   print([[
     <div style="height:90px;" class="jumbotron">
-      <h1 style="display:none">${opts.title}</h1>
+      <h1 style="display:none">${opts.plugin_global_title}</h1>
       <br>
-      <center class="col3">${opts.plugin_global_subtitle} </center>
+      <center class="col3">
+        ${opts.plugin_global_subtitle}
+        <a href="index.rss"><img src="img/rss.svg" class="icon rss"/></a>
+  ]])
+  if opts.plugin_rssatom_mastodonshare == 'enabled' then
+      print([[
+      <a class="script" onclick="alert('To start receiving the (RSS) feed (of this page),\nwe will guide you to create an activation-post'); document.location.href = 'https://' + prompt('Your mastodon instance:', 'mastodon.social') + '/share?text=follow+this+XRForge+instance+]] .. util.getURL('/') .. [[index.rss+via+@birb@rss-parrot.net'"><img src="img/mastodon.svg" class="icon mastodon"/></a>
+      ]])
+  end
+  print([[
+      </center>
     </div>
     <hr style="margin-top:0"/>
   ]])
@@ -13,8 +23,14 @@ end}
 
 {if parent ~= nil and parent.parent ~= nil then
   print([[
-    <h2><span class="col1">//</span> ${parent.name}</h2>
+  <h2><span class="col1">//</span> ${parent.name} <a href="${req.url}.rss"><img src="img/rss.svg" class="icon rss"/></a>
   ]])
+  if opts.plugin_rssatom_mastodonshare == 'enabled' then
+      print([[
+      <a class="script" onclick="alert('To start receiving the (RSS) feed (of this page),\nwe will guide you to create an activation-post'); document.location.href = 'https://' + prompt('Your mastodon instance:', 'mastodon.social') + '/share?text=follow+this+XRForge+instance+]] .. util.getURL() .. [[.rss+via+@birb@rss-parrot.net'"><img src="img/mastodon.svg" class="icon mastodon"/></a>
+      ]])
+  end
+  print('</h2>')
 end}
 
 <div class="grid">

src/page/dir.lua

@@ -7,13 +7,16 @@ return {
   middleware = function(req,res,next)
     local rootpath = "/" .. app.opts.plugin_diskscan_dirname 
     local urlpath  = req.url
-    local isroot   = urlpath == "/" or urlpath:match("^"..rootpath.."[/]?$") or urlpath == '/index.html'
+    local urlhash  = urlpath .. util.dump( req.param )
+    local isroot   = urlpath == "/" or urlpath:match("^"..rootpath.."[/]?$") or 
+                     urlpath == '/index.html' or urlpath == '/index.rss' 
+    local isRSS    = urlpath:match('%.rss$')
+    if isRSS then urlpath = urlpath:gsub("%.rss$","") end
     res.items      = {}
     res.template   = res.template or 'page/dir.html'
     local isdir    = urlpath:sub(-1) == "/" 
     local nslashes = util.countChar('/',urlpath)
     local match    = function(a,b) return util.stripPath(a) == util.stripPath(b) end
-    local urlhash  = urlpath .. util.dump( req.param )
     local referrer = util.baseURLify( (isroot and not req.param.url) and "" or urlpath )
 
     if (isroot or urlpath:match("^"..rootpath))
@@ -23,9 +26,11 @@ return {
       local redirect = false
       if not app.cache.url[ urlhash ] or util.count(req.params) > 0 then
         local matchpath = urlpath
-        if urlpath == '/' then matchpath = rootpath end -- generate janusxr footer for root url
+        -- generate janusxr footer / rss for root url
+        if urlpath == '/' or urlpath:match('index') then 
+          matchpath = rootpath 
+        end 
         foreach( app.cache.items, function(k,v)
-            print("URL=" ..v.url)
           if req.authenticated or app.plugin.moderation.isAllowed(v) then 
             if v.parent then
 --              trace( "path = "     .. util.stripPath(v.path) .. " " ..
@@ -95,6 +100,8 @@ return {
         end)
         if data.item then
           res.template = 'page/index.html'
+        elseif isRSS then
+          res.template = 'page/rss.xml'
         end
         app.pub("dir", nil, {res=res,data=data, urlpath=urlpath, isroot=isroot, isdir=isdir})
         if not redirect then
@@ -108,7 +115,7 @@ return {
         next()
       else 
         res.status(200)
-        res.header('content-type','text/html')
+        res.header('content-type', isRSS and 'application/rss+xml' or 'text/html')
         res.body( app.cache.url[ urlhash ] )
         next()
       end

src/page/header.html

@@ -6,6 +6,12 @@
     <base href="${opts.baseurl}" />
     <title>${opts.plugin_global_title}</title>
     <link rel="stylesheet" href="css/xrforge.css"></link>
+    <link 
+        rel="alternate" 
+        type="application/rss+xml" 
+        title="${opts.plugin_global_title} feed ${app.host}" 
+        href="${util.getURL('/')}index.rss"
+      />
     ${opts.plugin_global_header_html}
     <noscript>
       {if isroot then

src/page/rss.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:webfeeds="http://webfeeds.org/rss/1.0" xmlns:media="http://search.yahoo.com/mrss/">
+  <channel>
+    <title>${opts.plugin_global_title}</title>
+    <description>${opts.plugin_global_subtitle}</description>
+    <link>${app.scheme}://${app.host}${app.opts.baseurl}</link>
+    <image>
+      <url>${app.scheme}://${app.host}${app.opts.baseurl}img/backdrop.jpg</url>
+      <title>${opts.plugin_global_title}</title>
+      <link>${app.scheme}://${app.host}${app.opts.baseurl}</link>
+    </image>
+    {if true then
+      print('    <lastBuildDate>' .. FormatHttpDateTime(GetDate()) .. '</lastBuildDate>')
+    end}
+    <generator>XRForge</generator>
+ {if items then 
+
+    function getURL(url)
+      if app.arg[1] == 'generate' or url:match("://") then 
+        url = '?view=3D#janus.url=' .. EscapeFragment( url ) 
+        url = util.baseURLify(url)
+      else
+        url = util.stripPath( url )
+      end
+      return app.scheme .. '://' .. app.host .. app.opts.baseurl .. url
+    end
+
+    for i=1,#items do 
+      local item = items[i]
+      local visible = item.view2D == nil or (item.view2D ~= nil and item.view2D ~= false)
+      if visible then 
+        local thumbnail = ''
+        print([[
+          <item>
+            <guid isPermaLink="true">]] .. getURL(item.url) .. [[</guid>
+            <link>]] .. getURL(item.url) .. [[</link>
+            <pubDate>]] .. FormatHttpDateTime(GetDate()) .. [[</pubDate>
+            <description></description>
+            <media:content url="]] .. getURL(item.thumbnail) .. [[" medium="image">
+              <media:rating scheme="urn:simple">nonadult</media:rating>
+            </media:content>
+          </item>
+        ]])
+      end
+    end
+  end}
+
+  </channel>
+</rss>

src/plugin/archive.lua

@@ -1,4 +1,10 @@
 local util = require('util')
+local xml  = require('xmlSimple')
+app.on('init', function()
+  if app.opts.plugin_archive_scraper == 'enabled' then 
+    app.plugin.archive.scraper.start()
+  end
+end)
 
 app.on('scan_experience', function(exp)
   --print_r(exp)
@@ -10,25 +16,36 @@ end)
 app.on('set', function(k,v)
 end)
 
-app.on('init', function()
+app.on('scheduler', function(opts)
-  app.plugin.archive.waybackmachine()
+  if opts.interval == app.opts.plugin_archive_when then 
-end)
-
-app.on('autobackup', function()
     app.plugin.archive.waybackmachine()
+  end
 end)
 
 -- this registers the plugin UI
 local plugin = {
-  name = "<img src='img/drive.svg' class='icon invert'/> Archive.org",
+  name = "<img src='img/drive.svg' class='icon invert'/> Archiver",
-  info = 'Notify archive.org to archive experiences.<br><b>NOTE</b>: this only works in server mode. For static website generator-users: see <a href="https://web.archive.org/" target="_blank">Save Page Now</a>',
+  infowide = true,
+  info = [[
+    scrape / archive experiences via <a href="https://web.archive.org" target="_blank">waybackmachine</a> or to local disk.
+    <br><br>
+    <b>Recent log output:</b>
+    <br>
+    <iframe id="log" style="background: #88a; width: 100%;" frameborder="0" src="/scraper.txt"></iframe>
+    <br>
+    <script>
+      let log = document.querySelector('#log')
+      log.addEventListener('load', () => log.contentWindow.scrollTo(0,999999) )
+      var frameRefreshInterval = setInterval( () => log.src = log.src, 3000);
+    </script>
+    ]],
   opts = {
     { 
       key='app.opts.plugin_archive_when',
       title='When to trigger',
       info='when should archive.org check this xrforge?',
       default = "boot",
-      values={boot="On start", boot_autobackup="On start + autobackup event"}
+      values={disabled="disabled", daily="daily"}
     },
     { 
       key='app.opts.plugin_archive_url',
@@ -38,7 +55,39 @@ local plugin = {
       placeholder = "/myverse",
       values=''
     },
+    { 
+      key='app.opts.plugin_archive_scraper',
+      title='XR Scraper',
+      info='archive external links in experiences as local copies',
+      default = os.getenv('DEBUG') and 'enabled' or "disabled",
+      values={disabled="disabled",enabled="enabled"}
     },
+    { 
+      key='app.opts.plugin_archive_scraper_depth',
+      title='XR Scraper depth',
+      info='How deep should the spatial scraper go?',
+      default = "degree1",
+      values={degree1="1st degree outbound links",degree2="1st + 2nd degree outbound links"}
+    },
+    { 
+      key='app.opts.plugin_archive_scraper_rewritelinks',
+      title='XR Scraper proxy',
+      info='Local-first: serve scraped experiences instead of remote experiences (rewrites URLs)',
+      default = "enabled",
+      values={disabled="disabled",enabled="yes"}
+    }
+  },
+
+  url = {},
+
+  flush = function()
+    app.plugin.archive.url = {}
+  end,
+
+  add = function(url)
+    table.insert( app.plugin.archive.url, url )
+  end,
+
   waybackmachine = function()
     local url = 'https://web.archive.org/save/'.. app.scheme .. '://' .. app.host .. app.opts.plugin_archive_url 
     print("notifying archive.org: " .. url)
@@ -51,15 +100,60 @@ local plugin = {
     end
   end,
 
-  url = {},
+  scraper = {
 
-  flush = function()
+    start = function()
-    app.plugin.archive.url = {}
+      local me = app.plugin.archive.scraper
+      me.scraperStart = os.time()
+      util.fetchServer('/', nil, function(req,res)
+        me.scrapeLinks( req,res, 0 )
+      end)
     end,
 
-  add = function(url)
+    scrapeLinks = function(req, res, level)
-    table.insert( app.plugin.archive.url, url )
+      local me = app.plugin.archive.scraper
+      if level < 2 then 
+        foreach( me.scan, function( format, cb ) cb(req,res) end)
       end
+    end,
+
+    loadXML = function(xml,res,obj)
+      util.traverseXML( res.xml:ParseXmlText(xml),  function(node,raw)
+        if node.tag == 'link' and node.prop.url:match('[a-z]+://') then 
+          local newdir = util.stripPath( 
+                           app.opts.plugin_diskscan_dir .. '/' .. 
+                           node.prop.url:gsub(".*://",'')
+                                        :gsub("%?.*",'')           
+                                        :gsub("#.*",'')           
+                         )
+          if not util.fileExist( newdir ) then
+            util.execute('janusxr scrape "' .. node.prop.url .. '" "' .. newdir .. '" 1>&2')
+          else
+            print("already scraped: " .. node.prop.url )
+          end
+        end
+      end)
+    end,
+
+    scan = {
+
+      JML = function(req,res)
+        local me = app.plugin.archive.scraper
+        -- JML heuristic
+        if res.body:lower():match("<fireboxroom>") or res.body:lower():match("<room[ >]") then
+          local JML0
+          local JML
+          res.xml       = xml.newParser()
+          local xmlstr  = res.body
+          -- cleanup JML0
+          local JML0 = xmlstr:gsub(".*<[Rr]oom","<room")
+          JML0       = JML0:gsub("</[Rr]oom>.*","</room>")
+          me.loadXML( JML0, res, {} )
+        end
+      end
+    }
+
+  }
 }
 
 return plugin

src/plugin/clouddrive.lua

@@ -11,27 +11,7 @@ end)
 app.on('set', function(k,v)
 end)
 
-app.on('init', function()
+app.on('autobackup', function() -- triggered by src/plugin/webhook.lua
-  local me = app.plugin.archive
-  if arg[1] == 'server' then
-    local code, pid = unix.fork()
-    -- kill thread when main app gets CTRL-c
-    unix.sigaction(unix.SIGINT, function()
-      unix.exit()
-    end)
-    function autobackup()
-      if app.opts.plugin_global_backup_trigger == 'daily' then 
-        app.pub('autobackup', nil, {})
-        Sleep(60*24) -- 24 hours worth of seconds
-        autobackup() -- recurse forever
-      end
-      unix.exit(0)
-    end
-    if assert( code)  == 0 then autobackup() end -- go!
-  end
-end)
-
-app.on('autobackup', function()
   if util.cmdExist('rclone') then 
     print("running RCLONE cmd sync!")
   end

src/plugin/rssatom.lua

@@ -44,12 +44,19 @@ local plugin = {
       default="homepage",
       values={always='always',homepage='only homepage'}
     },
+    { 
+      key='app.opts.plugin_rssatom_mastodonshare',
+      title='Mastodon share',
+      info="Show <a href='https://mastodonshare.com' target='_blank'>mastodonshare.com</a> next to RSS-button",
+      default="enabled",
+      values={enabled='enabled',disabled='disabled'}
+    },
     { 
       key='app.opts.plugin_rssatom_url_1',
       title='RSS/Atom URL 1',
       info="URL and/or Thumbnail URL (space separated)",
-      default="http://mastodon.online/@lvk.rss https://pbs.twimg.com/profile_images/1604566357206601730/dlRNo-V-_400x400.jpg",
+      default="https://mastodon.online/@lvk.rss https://pbs.twimg.com/profile_images/1604566357206601730/dlRNo-V-_400x400.jpg",
-      values='http://mastodon.online/@lvk.rss https://pbs.twimg.com/profile_images/1604566357206601730/dlRNo-V-_400x400.jpg'
+      values='https://mastodon.online/@lvk.rss https://pbs.twimg.com/profile_images/1604566357206601730/dlRNo-V-_400x400.jpg'
     },
     { 
       key='app.opts.plugin_rssatom_url_2',

src/plugin/scheduler.lua

@@ -0,0 +1,54 @@
+app.on('init', function()
+  app.plugin.scheduler.start()
+end)
+
+
+return {
+
+  name = 'Scheduler',
+  hide = true,
+
+  start = function()
+    local me = app.plugin.scheduler
+    if arg[1] == 'server' then
+      local pid, err = unix.fork()
+      -- kill thread when main app gets CTRL-c
+      if pid == 0 then -- child has zero pid
+        function periodically()
+          local check_step = os.getenv('DEBUG') and 1 or 5
+          local elapsed = { min5 = 0, hourly = 0, daily = 0, daily2 = 0, weekly = 0}
+          local intervals = {
+            min5    = 60*5,       -- 5 mins worth seconds
+            hourly  = 60*60,      -- 1 hours worth of seconds
+            daily   = 60*60*24,   -- 24 hours worth of seconds
+            daily2  = 60*60*24*2, -- 24*2 hours worth of seconds
+            weekly  = 60*60*24*7  -- 24*7 hours worth of seconds
+          }
+          Sleep(check_step)
+          foreach( elapsed, function(interval,time)
+            elapsed[interval] = time + check_step
+            if elapsed[interval] > intervals[interval] 
+              or os.getenv('DEBUG') 
+            then 
+              trace("scheduler.lua: debug-event: 'scheduler' " .. interval)
+              app.pub('scheduler', nil, {interval=interval})
+              elapsed[interval]= 0 -- reset
+            end
+          end)
+          if not os.getenv('DEBUG') then 
+            periodically() -- recurse forever
+          end
+        end
+        periodically() -- go!
+      else
+        -- main thread gets non-zero pid (of child) 
+        unix.sigaction(unix.SIGINT, function()
+          if pid ~= nil then
+            unix.kill( pid, unix.SIGTERM)
+          end
+          unix.exit()
+        end)
+      end 
+    end
+  end
+}

src/plugin/staticwebgen.lua

@@ -65,7 +65,7 @@ local plugin = {
   generate = function()
     local outdir = 'www'
     local root   = app.opts.plugin_diskscan_dirname
-    local urls = {"/","/about", "/" .. root }
+    local urls = {"/","/about", "/" .. root, "/index.rss" }
     local urlLast = ''
     if util.fileExist(outdir) then 
       print("[!] folder 'www' was already detected..please delete it")
@@ -77,6 +77,7 @@ local plugin = {
     os.execute("cd " .. outdir .. " && unzip " .. selfbinary .. " 'img/*' 'css/*' 'janusweb/*' 'shader/*'")
     foreach( app.cache.items, function(k,v) 
       if v.dir then 
+        table.insert(urls, v.url .. ".rss" )
         table.insert(urls, v.url )
         urlLast = v.url
       end

src/plugin/webhook.lua

@@ -50,6 +50,13 @@ local plugin = {
       info='URL to trigger when changes occur',
       default="",
       values=''
+    },
+    { 
+      key='app.opts.plugin_event_autobackup',
+      title="Event 'autobackup'",
+      info='Trigger frequency',
+      default="daily",
+      values={min5="every 5 mins",hourly="every hour",daily="every day"}
     }
   }
 }

src/util/build.sh

@@ -52,7 +52,7 @@ build(){
 
 rebuild(){
   cd src
-  zip -yr ../result/bin/xrforge.com .lua shader img middlware page plugin cmd css .init.lua .args test myverse
+  zip -yr ../result/bin/xrforge.com .lua js shader img middlware page plugin cmd css .init.lua .args test myverse
 }
 
 "$@"