implemented 'local avatars', downloading images, and disabling hotlinks

README.md

@@ -11,7 +11,14 @@ The filter uses the variable `pleroma-urls` for input which can be defined in th
 
 * `pleroma-urls`: a list of strings that should contain the URLs of the posts to source replies from.
 
-The filter exports variables that can be used in a pandoc template.
+The following two are optional.  If both are defined, then the script will download avatars and will not use hotlinks. 
+
+1. (lua) `pleroma_avatar_path` = (yaml) `pleroma-avatar-path`: a string ending in `/`, the folder containing avatars as referenced by the web page (e.g `<img src="(pleroma-avatar-path)/avatar.png"/>`).
+2. (lua) `pleroma_avatar_save_path` = (yaml) `pleroma-avatar-save-path`: a string ending in `/`, the folder in which avatars are downloaded to.
+
+If multiple pages will be storing avatars in the same folder, create `config.lua` and assign the variables there. If a specific page will have its own directory for images, assign the variables in the manuscript's YAML header.
+
+The filter exports variables that can be used in a Pandoc template.
 
 * `pleroma-comments`: string, the full comments section rendered by the script.
 * `pleroma-comments-count`: integer, the number of comments received by the script.

pleroma-comments.lua

@@ -1,15 +1,51 @@
 
--- can't resolve seggfault on luasec, luasocket'
--- local https = "wget"
 local json = require("cjson")
 
+pleroma_avatar_save_path = nil  -- location of downloaded images
+pleroma_avatar_path = nil       -- prepended to image file, written on page
+-- to configure, put above vars in a file named 'config.lua'
+pcall(
+    function () 
+        dofile("config.lua")
+    end
+)
+--[[
+
+if these variables are still `nil` after `dofile()` then
+the script will look for it in the yaml header.
+
+note: lua filters can not access variables defined on the command line,
+e.g `pandoc -V foo="bar"`
+and it may not be ideal to define paths in yaml headers,
+i.e changing the path for avatars on a website would involve editing each 
+manuscript.
+
+]]--
+
 -- for testing 
 function printTable(t)
-    for key, value in pairs(t) do
+    if t then
-        print(key, value)
+        for key, value in pairs(t) do
+            print(key, value)
+        end
     end
 end
 
+function add_unique(list, item)
+    if type(list) ~= "table" or item == nil then
+--         print("item "..item)
+        return false
+    end
+    for _, value in ipairs(list) do 
+        if value == item then
+            return false
+        end
+    end
+    table.insert(list, item)
+--     print("added "..item)
+    return true
+end
+
 function tokenizeString(inputString, delimiter)
     local tokens = {}
     for token in inputString:gmatch("[^" .. delimiter .. "]+") do
@@ -18,15 +54,36 @@ function tokenizeString(inputString, delimiter)
     return tokens
 end
 
-function get(link)
+function get(link, filename)
-    local args = {
+    print("http/s GET: ".. link)
-        "-qO-",
+    local filename = filename or nil
-        link
+    local args = {}
-    }
+    if filename then
-    local data = pandoc.pipe("wget", args, "")
+        args = {
-    local parsed = json.decode(data)
+            "--timeout=10",
---     print(link)
+            "-qO",
-    return parsed
+            filename,
+            link
+        }
+    else
+        args = {
+            "-qO-",
+            "--timeout=10",
+            link
+        }
+    end
+    local command = "wget " .. table.concat(args, ' ')
+    local success, retval = pcall(
+        function () 
+--             print("!: ".. table.concat(args))
+            return pandoc.pipe("wget", args, "") 
+        end
+    )
+    if not success then
+        print("warning: error while performing http/s GET")
+        print(retval)
+    end
+    return retval
 end
 
 function get_epoch_time(timestamp)
@@ -49,33 +106,55 @@ function get_short_date(timestamp)
     )
 end
 
-function write_comments(pleroma_posts, instance, show_avatars)
+function write_comments(pleroma_posts, instance, avatar_path)
-    show_avatars = show_avatars or false
+    local avatar_mode = 1
+    if avatar_path then
+        avatar_mode = 2
+    end
 
-    function get_user(acct_data, instance, include_img)
+    -- img mode : 0 = omit; 1 = display (hotlink); 2 = download
-        -- user data
+    function get_user(acct_data, instance, img_mode, folder)
-        local user_info = ""
+        -- specify path to store avatars
+        -- should be empty string if img_mode != 2
+        local folder = folder or ""
+
+        -- related to output
+        local user_info = "" -- template
         local result = ""
+
         local vars = {
             alias = acct_data["display_name"],
             uid = acct_data["id"],
             handle = acct_data["acct"],
             host=instance
         }
-        if include_img then 
+        local filename = nil
+        local avatar_url = acct_data["avatar_static"]
+        if img_mode then 
             user_info = [[
             <figure>
                 <img src="$avatar$" loading="lazy" alt="avatar"/>
                 <figcaption>$alias$ <a href="$host$/users/$uid$">@$handle$</a> </figcaption>
             </figure>
             ]]
-            vars.avatar = acct_data["avatar_static"]
+            if img_mode == 1 then 
+                vars.avatar = acct_data["avatar_static"]
+            else
+--                 save to file such as user_at_instance_tld.png
+                -- omit query e.g ?name="foo" - get the extension only
+                local extension = (avatar_url:match("([^?]+)")):match("^.+(%..+)$")
+                -- replace '@'s and '.'
+                local name = (acct_data["acct"]:gsub("@", "_at_")):gsub("%.", "_")
+                filename = name .. extension
+                vars.avatar = folder .. filename
+            end
             result = user_info:gsub("%$(%w+)%$", vars)
         else
-            user_info = "<p>$alias$ <a href=\"$host$/users/$uid$\">@$handle$</a></p>"
+            user_info = "<p>$a`lias$ <a href=\"$host$/users/$uid$\">@$handle$</a></p>"
             result = user_info:gsub("%$(%w+)%$", vars)
         end
-        return result
+--         print("vars: " .. vars.avatar)
+        return result, filename, avatar_url
     end
 
     function get_card(card, instance)
@@ -124,7 +203,7 @@ function write_comments(pleroma_posts, instance, show_avatars)
                 mime = v["pleroma"]["mime_type"]
             }
             local foo = item:gsub("%$(%w+)%$", vars)
-            print(foo)
+--             print(foo)
             table.insert(media_list, foo)
         end
         table.insert(media_list, "</ol>")
@@ -188,6 +267,10 @@ function write_comments(pleroma_posts, instance, show_avatars)
 ]]
     local comments = {}
     local replies = pleroma_posts-- ["descendants"]
+
+    local links = {}
+    local images = {}
+
     for i, post in ipairs(replies) do
         local pid = post["id"]
         local datetime = get_short_date(post["created_at"])
@@ -199,12 +282,17 @@ function write_comments(pleroma_posts, instance, show_avatars)
         )
         table.insert(attrs, get_poll(post["poll"]))
 
+        local user, img_file, img_url = get_user(
+            post["account"], instance, avatar_mode, avatar_path)
+        add_unique(images, img_file)
+        add_unique(links, img_url)
+
         local interpolated = template:gsub("%$(%w+)%$", {
             i= #replies - i + 1,
             host=instance,
             pid=pid,
             datetime=datetime,
-            user=get_user(post["account"], instance, true),
+            user=user,
             text = text,
             card = get_card(post["card"], instance),
             attributes = table.concat(attrs)
@@ -214,8 +302,8 @@ function write_comments(pleroma_posts, instance, show_avatars)
             comments, pandoc.RawBlock("html", interpolated)
         )
     end
---     print(comments)
+--     printTable(dl_list)
-    return comments
+    return comments, images, links
 end
 
 function combine_tables(a,b)
@@ -242,23 +330,59 @@ end
 
 function get_status(host, post_id)
     local url = "https://" .. host .. "/api/v1/statuses/" .. post_id
-    print(url)
+--     print(url)
-    return get(url)
+    return json.decode(get(url))
 end
 
 function get_replies(host, id)
     local url = "https://" .. host ..  "/api/v1/statuses/" .. id .. "/context"
-    print(url)
+--     print(url)
-    local got = get(url)
+    local got = json.decode(get(url))
     return got["descendants"]
 end
 
+function get_images(filenames, urls, folder)
+    if not folder then
+        folder = ""
+    end
+    if not filenames or not urls then
+        return
+    end
+    if #filenames ~= #urls then
+        return
+    end
+    for i = 1, #urls, 1 do 
+        -- still possible to have a ilst of nil (file names)
+        if not filenames[i]  then
+            break
+        end
+        get(urls[i], folder .. filenames[i])
+    end
+end
+
 function Meta(meta)
     local pleroma_urls = meta["pleroma-urls"]
---     print(elem)
     if pleroma_urls == nil then
         return -- abort
     end
+
+    -- if both are defined, then do not hotlink avatars
+    if not pleroma_avatar_save_path then
+        pleroma_avatar_save_path = meta["pleroma-avatar-save-path"]
+    end
+    if not pleroma_avatar_path then
+        pleroma_avatar_path = meta["pleroma-avatar-path"]
+    end
+    -- most servers appear to serve hotilnked avatars however
+    -- images will be missing in case of downtime or shutdown
+    -- OR a user has changed their avatar and the old avatar gets deleted
+
+    -- var currently unused
+--     local is_hotlink = true 
+--     if pleroma_avatar_save_path and pleroma_avatar_path then
+--         is_hotlink = false
+--     end
+
     local all_replies = {}
     local hrefs = {}
     local host = ""
@@ -281,7 +405,13 @@ function Meta(meta)
             return ta > tb
         end
     )
-    local c = write_comments(all_replies, "https://" .. host)
+    -- returns comments, images, links (img urls)
+    local c, i, l = write_comments(
+        all_replies, 
+        "https://" .. host,
+        pleroma_avatar_path
+    )
+    get_images(i, l, pleroma_avatar_save_path)
     meta["pleroma-comments"] = c
     meta["pleroma-comments-count"] = #c
     meta["pleroma-has-comments"] = (#c > 0)