aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--serialize_lib/atomic.lua217
-rw-r--r--serialize_lib/init.lua74
-rw-r--r--serialize_lib/mod.conf4
-rw-r--r--serialize_lib/readme.md7
-rw-r--r--serialize_lib/serialize.lua269
-rw-r--r--serialize_lib/settingtypes.txt12
-rw-r--r--serialize_lib/tests/serialize_spec.lua123
7 files changed, 706 insertions, 0 deletions
diff --git a/serialize_lib/atomic.lua b/serialize_lib/atomic.lua
new file mode 100644
index 0000000..182cf42
--- /dev/null
+++ b/serialize_lib/atomic.lua
@@ -0,0 +1,217 @@
+-- atomic.lua
+-- Utilities for transaction-like handling of serialized state files
+-- Also for multiple files that must be synchronous, as advtrains currently requires.
+
+
+-- Managing files and backups
+-- ==========================
+
+--[[
+The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within
+the write, so we have incomplete data. So, the following scheme is applied:
+Unix:
+1. writes to <filename>.new
+2. moves <filename>.new to <filename>, clobbering previous file
+Windows:
+1. writes to <filename>.new
+2. delete <filename>
+3. moves <filename>.new to <filename>
+
+We count a new version of the state as "committed" after stage 2.
+
+During loading, we apply the following order of precedence:
+1. <filename>
+2. <filename>.new (windows only, in case we were interrupted just before 3. when saving)
+
+
+All of these functions return either true on success or nil, error on error.
+]]--
+
+local ser = serialize_lib.serialize
+
+local windows_mode = false
+
+-- == local functions ==
+
+local function save_atomic_move_file(filename)
+ --2. if windows mode, delete main file
+ if windows_mode then
+ local delsucc, err = os.remove(filename)
+ if not delsucc then
+ serialize_lib.log_warn("Unable to delete old savefile '"..filename.."':")
+ serialize_lib.log_warn(err)
+ serialize_lib.log_info("Trying to replace the save file anyway now...")
+ end
+ end
+
+ --3. move file
+ local mvsucc, err = os.rename(filename..".new", filename)
+ if not mvsucc then
+ if minetest.settings:get_bool("serialize_lib_no_auto_windows_mode") or windows_mode then
+ serialize_lib.log_error("Unable to replace save file '"..filename.."':")
+ serialize_lib.log_error(err)
+ return nil, err
+ else
+ -- enable windows mode and try again
+ serialize_lib.log_info("Unable to replace save file '"..filename.."' by direct renaming:")
+ serialize_lib.log_info(err)
+ serialize_lib.log_info("Enabling Windows mode for atomic saving...")
+ windows_mode = true
+ return save_atomic_move_file(filename)
+ end
+ end
+
+ return true
+end
+
+local function open_file_and_save_callback(callback, filename)
+ local file, err = io.open(filename, "w")
+ if not file then
+ error("Failed opening file '"..filename.."' for write:\n"..err)
+ end
+
+ callback(file)
+ return true
+end
+
+local function open_file_and_load_callback(filename, callback)
+ local file, err = io.open(filename, "r")
+ if not file then
+ error("Failed opening file '"..filename.."' for read:\n"..err)
+ end
+
+ return callback(file)
+end
+
+-- == public functions ==
+
+-- Load a saved state (according to comment above)
+-- if 'callback' is nil: reads serialized table.
+-- returns the read table, or nil,err on error
+-- if 'callback' is a function (signature func(file_handle) ):
+-- Counterpart to save_atomic with function argument. Opens the file and calls callback on it.
+-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
+-- The callback's first return value is returned by load_atomic
+function serialize_lib.load_atomic(filename, callback)
+
+ local cbfunc = callback or ser.read_from_fd
+
+ -- try <filename>
+ local file, ret = io.open(filename, "r")
+ if file then
+ -- read the file using the callback
+ local success
+ success, ret = pcall(cbfunc, file)
+ if success then
+ return ret
+ end
+ end
+
+ if minetest.settings:get_bool("serialize_lib_strict_loading") then
+ serialize_lib.save_lock = true
+ error("Loading data from file '"..filename.."' failed:\n"
+ ..ret.."\nDisable Strict Loading to ignore.")
+ end
+
+ serialize_lib.log_warn("Loading data from file '"..filename.."' failed, trying .new fallback:")
+ serialize_lib.log_warn(ret)
+
+ -- try <filename>.new
+ file, ret = io.open(filename..".new", "r")
+ if file then
+ -- read the file using the callback
+ local success
+ success, ret = pcall(cbfunc, file)
+ if success then
+ return ret
+ end
+ end
+
+ serialize_lib.log_error("Unable to load data from '"..filename..".new':")
+ serialize_lib.log_error(ret)
+ serialize_lib.log_error("Note: This message is normal when the mod is loaded the first time on this world.")
+
+ return nil, ret
+end
+
+-- Save a file atomically (as described above)
+-- 'data' is the data to be saved (when a callback is used, this can be nil)
+-- if 'callback' is nil:
+-- data must be a table, and is serialized into the file
+-- if 'callback' is a function (signature func(data, file_handle) ):
+-- Opens the file and calls callback on it. The 'data' argument is the data passed to save_atomic().
+-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
+-- The callback's first return value is returned by load_atomic
+-- Important: the callback must close the file in all cases!
+function serialize_lib.save_atomic(data, filename, callback, config)
+ if serialize_lib.save_lock then
+ serialize_lib.log_warn("Instructed to save '"..filename.."', but save lock is active!")
+ return nil
+ end
+
+ local cbfunc = callback or ser.write_to_fd
+
+ local file, ret = io.open(filename..".new", "w")
+ if file then
+ -- save the file using the callback
+ local success
+ success, ret = pcall(cbfunc, data, file)
+ if success then
+ return save_atomic_move_file(filename)
+ end
+ end
+ serialize_lib.log_error("Unable to save data to '"..filename..".new':")
+ serialize_lib.log_error(ret)
+ return nil, ret
+end
+
+
+-- Saves multiple files synchronously. First writes all data to all <filename>.new files,
+-- then moves all files in quick succession to avoid inconsistent backups.
+-- parts_table is a table where the keys are used as part of the filename and the values
+-- are the respective data written to it.
+-- e.g. if parts_table={foo={...}, bar={...}}, then <filename_prefix>foo and <filename_prefix>bar are written out.
+-- if 'callbacks_table' is defined, it is consulted for callbacks the same way save_atomic does.
+-- example: if callbacks_table = {foo = func()...}, then the callback is used during writing of file 'foo' (but not for 'bar')
+-- Note however that you must at least insert a "true" in the parts_table if you don't use the data argument.
+-- Important: the callback must close the file in all cases!
+function serialize_lib.save_atomic_multiple(parts_table, filename_prefix, callbacks_table, config)
+ if serialize_lib.save_lock then
+ serialize_lib.log_warn("Instructed to save '"..filename_prefix.."' (multiple), but save lock is active!")
+ return nil
+ end
+
+ for subfile, data in pairs(parts_table) do
+ local filename = filename_prefix..subfile
+ local cbfunc = ser.write_to_fd
+ if callbacks_table and callbacks_table[subfile] then
+ cbfunc = callbacks_table[subfile]
+ end
+
+ local success = false
+ local file, ret = io.open(filename..".new", "w")
+ if file then
+ -- save the file using the callback
+ success, ret = pcall(cbfunc, data, file, config)
+ end
+
+ if not success then
+ serialize_lib.log_error("Unable to save data to '"..filename..".new':")
+ serialize_lib.log_error(ret)
+ return nil, ret
+ end
+ end
+
+ local first_error
+ for file, _ in pairs(parts_table) do
+ local filename = filename_prefix..file
+ local succ, err = save_atomic_move_file(filename)
+ if not succ and not first_error then
+ first_error = err
+ end
+ end
+
+ return not first_error, first_error -- either true,nil or nil,error
+end
+
+
diff --git a/serialize_lib/init.lua b/serialize_lib/init.lua
new file mode 100644
index 0000000..20ffa4d
--- /dev/null
+++ b/serialize_lib/init.lua
@@ -0,0 +1,74 @@
+-- serialize_lib
+--[[
+ Copyright (C) 2020 Moritz Blei (orwell96) and contributors
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as
+ published by the Free Software Foundation, either version 3 of the
+ License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+]]--
+
+serialize_lib = {}
+
+--[[ Configuration table
+Whenever asked for a "config", the following table structure is expected:
+config = {
+ skip_empty_tables = false -- if true, does not store empty tables
+ -- On next read, keys that mapped to empty tables resolve to nil
+ -- Used by: write_table_to_file
+}
+Not all functions use all of the parameters, so you can simplify your config sometimes
+]]
+
+-- log utils
+-- =========
+
+
+function serialize_lib.log_error(text)
+ minetest.log("error", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
+end
+function serialize_lib.log_warn(text)
+ minetest.log("warning", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
+end
+function serialize_lib.log_info(text)
+ minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
+end
+function serialize_lib.log_debug(text)
+ minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?")..") DEBUG: "..(text or "<nil>"))
+end
+
+-- basic serialization/deserialization
+-- ===================================
+
+local mp = minetest.get_modpath(minetest.get_current_modname())
+serialize_lib.serialize = dofile(mp.."/serialize.lua")
+dofile(mp.."/atomic.lua")
+
+local ser = serialize_lib.serialize
+
+-- Opens the passed filename, and returns deserialized table
+-- When an error occurs, logs an error and returns false
+function serialize_lib.read_table_from_file(filename)
+ local succ, ret = pcall(ser.read_from_file, filename)
+ if not succ then
+ serialize_lib.log_error(ret)
+ end
+ return ret
+end
+
+-- Writes table into file
+-- When an error occurs, logs an error and returns false
+function serialize_lib.write_table_to_file(root_table, filename)
+ local succ, ret = pcall(ser.write_to_file, root_table, filename)
+ if not succ then
+ serialize_lib.log_error(ret)
+ end
+ return ret
+end
+
+
diff --git a/serialize_lib/mod.conf b/serialize_lib/mod.conf
new file mode 100644
index 0000000..a2fd6bb
--- /dev/null
+++ b/serialize_lib/mod.conf
@@ -0,0 +1,4 @@
+name=serialize_lib
+title=Lua Serialization Library
+description=A library to efficiently write and read Lua tables to/from a file, with support for atomic operations
+author=orwell96
diff --git a/serialize_lib/readme.md b/serialize_lib/readme.md
new file mode 100644
index 0000000..ac364b8
--- /dev/null
+++ b/serialize_lib/readme.md
@@ -0,0 +1,7 @@
+# serialize_lib
+A Minetest mod library for safely storing large amounts of data in on-disk files.
+Created out of the need to have a robust data store for advtrains.
+
+The main purpose is to load and store large Lua table structures into files, without loading everything in memory and exhausting the function constant limit of LuaJIT.
+
+Also contains various utilities to handle files on disk in a safe manner, retain multiple versions of the same file a.s.o. \ No newline at end of file
diff --git a/serialize_lib/serialize.lua b/serialize_lib/serialize.lua
new file mode 100644
index 0000000..12a26c4
--- /dev/null
+++ b/serialize_lib/serialize.lua
@@ -0,0 +1,269 @@
+-- serialize.lua
+-- Lua-conformant library file that has no minetest dependencies
+-- Contains the serialization and deserialization routines
+
+--[[
+
+Structure of entry:
+[keytype][key]:[valuetype][val]
+Types:
+ B - bool
+ -> 0=false, 1=true
+ S - string
+ -> see below
+ N - number
+ -> thing compatible with tonumber()
+Table:
+[keytype][key]:T
+... content is nested in table until the matching
+E
+
+example:
+LUA_SER v=1 {
+Skey:Svalue key = "value",
+N1:Seins [1] = "eins",
+B1:T [true] = {
+Sa:Sb a = "b",
+Sc:B0 c = false,
+E }
+E }
+
+String representations:
+In strings the following characters are escaped by &
+'&' -> '&&'
+(line break) -> '&n'
+(CR) -> '&r' --> required?
+':' -> '&:'
+All other characters are unchanged as they bear no special meaning.
+]]
+
+local write_table, literal_to_string, escape_chars, table_is_empty
+
+function table_is_empty(t)
+ for _,_ in pairs(t) do
+ return false
+ end
+ return true
+end
+
+function write_table(t, file, config)
+ local ks, vs, writeit, istable
+ for key, value in pairs(t) do
+ ks = value_to_string(key, false)
+ writeit = true
+ istable = type(value)=="table"
+
+ if istable then
+ vs = "T"
+ if config and config.skip_empty_tables then
+ writeit = not table_is_empty(value)
+ end
+ else
+ vs = value_to_string(value, true)
+ end
+
+ if writeit then
+ file:write(ks..":"..vs.."\n")
+
+ if istable then
+ write_table(value, file, config)
+ file:write("E\n")
+ end
+ end
+ end
+end
+
+function value_to_string(t)
+ if type(t)=="table" then
+ file:close()
+ error("Can not serialize a table in the key position!")
+ elseif type(t)=="boolean" then
+ if t then
+ return "B1"
+ else
+ return "B0"
+ end
+ elseif type(t)=="number" then
+ return "N"..t
+ elseif type(t)=="string" then
+ return "S"..escape_chars(t)
+ else
+ --error("Can not serialize '"..type(t).."' type!")
+ return "S<function>"
+ end
+ return str
+end
+
+function escape_chars(str)
+ local rstr = string.gsub(str, "&", "&&")
+ rstr = string.gsub(rstr, ":", "&:")
+ rstr = string.gsub(rstr, "\n", "&n")
+ return rstr
+end
+
+------
+
+local read_table, string_to_value, unescape_chars
+
+function read_table(t, file)
+ local line, ks, vs, kv, vv, vt
+ while true do
+ line = file:read("*l")
+ if not line then
+ file:close()
+ error("Unexpected EOF or read error!")
+ end
+
+ if line=="E" then
+ -- done with this table
+ return
+ end
+ ks, vs = string.match(line, "^(.*[^&]):(.+)$")
+ if not ks or not vs then
+ file:close()
+ error("Unable to parse line: '"..line.."'!")
+ end
+ kv = string_to_value(ks)
+ vv, vt = string_to_value(vs, true)
+ if vt then
+ read_table(vv, file)
+ end
+ -- put read value in table
+ t[kv] = vv
+ end
+end
+
+-- returns: value, is_table
+function string_to_value(str, table_allow)
+ local first = string.sub(str, 1,1)
+ local rest = string.sub(str, 2)
+ if first=="T" then
+ if table_allow then
+ return {}, true
+ else
+ file:close()
+ error("Table not allowed in key component!")
+ end
+ elseif first=="N" then
+ local num = tonumber(rest)
+ if num then
+ return num
+ else
+ file:close()
+ error("Unable to parse number: '"..rest.."'!")
+ end
+ elseif first=="B" then
+ if rest=="0" then
+ return false
+ elseif rest=="1" then
+ return true
+ else
+ file:close()
+ error("Unable to parse boolean: '"..rest.."'!")
+ end
+ elseif first=="S" then
+ return unescape_chars(rest)
+ else
+ file:close()
+ error("Unknown literal type '"..first.."' for literal '"..str.."'!")
+ end
+end
+
+function unescape_chars(str) --TODO
+ local rstr = string.gsub(str, "&:", ":")
+ rstr = string.gsub(rstr, "&n", "\n")
+ rstr = string.gsub(rstr, "&&", "&")
+ return rstr
+end
+
+------
+
+--[[
+config = {
+ skip_empty_tables = false -- if true, does not store empty tables
+ -- On next read, keys that mapped to empty tables resolve to nil
+}
+]]
+
+-- Writes the passed table into the passed file descriptor, and closes the file
+local function write_to_fd(root_table, file, config)
+ file:write("LUA_SER v=1\n")
+ write_table(root_table, file, config)
+ file:write("E\nEND_SER\n")
+ file:close()
+end
+
+-- Reads the file contents from the passed file descriptor and returns the table on success
+-- Throws errors when something is wrong. Closes the file.
+-- config: see above
+local function read_from_fd(file)
+ local first_line = file:read("*l")
+ if first_line ~= "LUA_SER v=1" then
+ file:close()
+ error("Expected header, got '"..first_line.."' instead!")
+ end
+ local t = {}
+ read_table(t, file)
+ local last_line = file:read("*l")
+ file:close()
+ if last_line ~= "END_SER" then
+ error("Missing END_SER, got '"..last_line.."' instead!")
+ end
+ return t
+end
+
+-- Opens the passed filename and serializes root_table into it
+-- config: see above
+function write_to_file(root_table, filename, config)
+ -- try opening the file
+ local file, err = io.open(filename, "w")
+ if not file then
+ error("Failed opening file '"..filename.."' for write:\n"..err)
+ end
+
+ write_to_fd(root_table, file, config)
+ return true
+end
+
+-- Opens the passed filename, and returns its deserialized contents
+function read_from_file(filename)
+ -- try opening the file
+ local file, err = io.open(filename, "r")
+ if not file then
+ error("Failed opening file '"..filename.."' for read:\n"..err)
+ end
+
+ return read_from_fd(file)
+end
+
+--[[ simple unit test
+local testtable = {
+ key = "value",
+ [1] = "eins",
+ [true] = {
+ a = "b",
+ c = false,
+ },
+ ["es:cape1"] = "foo:bar",
+ ["es&ca\npe2"] = "baz&bam\nbim",
+ ["es&&ca&\npe3"] = "baz&&bam&\nbim",
+ ["es&:cape4"] = "foo\n:bar"
+}
+local config = {}
+--write_to_file(testtable, "test_out", config)
+local t = read_from_file("test_out")
+write_to_file(t, "test_out_2", config)
+local t2 = read_from_file("test_out_2")
+write_to_file(t2, "test_out_3", config)
+
+-- test_out_2 and test_out_3 should be equal
+
+--]]
+
+
+return {
+ read_from_fd = read_from_fd,
+ write_to_fd = write_to_fd,
+ read_from_file = read_from_file,
+ write_to_file = write_to_file,
+}
diff --git a/serialize_lib/settingtypes.txt b/serialize_lib/settingtypes.txt
new file mode 100644
index 0000000..3a565a6
--- /dev/null
+++ b/serialize_lib/settingtypes.txt
@@ -0,0 +1,12 @@
+# Enable strict file loading mode
+# If enabled, if any error occurs during loading of a file using the 'atomic' API,
+# an error is thrown. You probably need to disable this option for initial loading after
+# creating the world.
+serialize_lib_strict_loading (Strict loading) bool false
+
+# Do not automatically switch to "Windows mode" when saving atomically
+# Normally, when renaming <filename>.new to <filename> fails, serialize_lib automatically
+# switches to a mode where it deletes <filename> prior to moving. Enable this option to prevent
+# this behavior and abort saving instead.
+serialize_lib_no_auto_windows_mode (No automatic Windows Mode) bool false
+
diff --git a/serialize_lib/tests/serialize_spec.lua b/serialize_lib/tests/serialize_spec.lua
new file mode 100644
index 0000000..ccc3a67
--- /dev/null
+++ b/serialize_lib/tests/serialize_spec.lua
@@ -0,0 +1,123 @@
+-- test the serialization function
+
+
+package.path = "../?.lua;" .. package.path
+
+
+ser = require("serialize")
+
+
+local mock_file = {}
+_G.mock_file = mock_file
+function mock_file:read(arg)
+ if arg == "*l" then
+ local l = self.lines[self.pointer or 1]
+ self.pointer = (self.pointer or 1) + 1
+ return l
+ end
+end
+
+function mock_file:close()
+ return nil
+end
+
+function mock_file:write(text)
+ self.content = self.content..text
+end
+
+function mock_file:create(lines)
+ local f = {}
+ setmetatable(f, mock_file)
+ f.lines = lines or {}
+ f.write = self.write
+ f.close = self.close
+ f.read = self.read
+ f.content = ""
+ return f
+end
+
+
+local testtable = {
+ key = "value",
+ [1] = "eins",
+ [true] = {
+ a = "b",
+ c = false,
+ },
+ ["es:cape1"] = "foo:bar",
+ ["es&ca\npe2"] = "baz&bam\nbim",
+ ["es&&ca&\npe3"] = "baz&&bam&\nbim",
+ ["es&:cape4"] = "foo\n:bar"
+}
+local testser = [[LUA_SER v=1
+B1:T
+Sa:Sb
+Sc:B0
+E
+Skey:Svalue
+Ses&&&&ca&&&npe3:Sbaz&&&&bam&&&nbim
+N1:Seins
+Ses&&&:cape4:Sfoo&n&:bar
+Ses&&ca&npe2:Sbaz&&bam&nbim
+Ses&:cape1:Sfoo&:bar
+E
+END_SER
+]]
+
+local function check_write(tb, conf)
+ f = mock_file:create()
+ ser.write_to_fd(tb, f, conf or {})
+ return f.content
+end
+
+function string:split()
+ local fields = {}
+ self:gsub("[^\n]+", function(c) fields[#fields+1] = c end)
+ return fields
+end
+
+local function check_read(text)
+ f = mock_file:create(text:split())
+ return ser.read_from_fd(f)
+end
+
+local noskip = [[LUA_SER v=1
+N1:T
+E
+E
+END_SER
+]]
+local skip = [[LUA_SER v=1
+E
+END_SER
+]]
+
+describe("write_to_fd", function()
+ it("serializes a table correctly", function()
+ assert.equals(check_write(testtable), testser)
+ end)
+ it("does not skip empty tables", function()
+ assert.equals(check_write({{}}),noskip)
+ end)
+ it("skips empty tables when needed", function()
+
+ assert.equals(check_write({{}},{skip_empty_tables=true}),skip)
+ end)
+end)
+
+describe("read_from_fd", function ()
+ it("reads a table correctly", function()
+ assert.same(check_read(testser),testtable)
+ end)
+ it("handles some edge cases correctly", function()
+ assert.same(check_read(noskip), {{}})
+ assert.same(check_read(skip), {})
+ end)
+ it("Read back table", function()
+ local tb = {}
+ for k=1,262 do
+ tb[k] = { "Foo", "bar", k}
+ end
+ assert.same(check_read(check_write(tb)), tb)
+ end)
+end)