diff options
| -rw-r--r-- | serialize_lib/atomic.lua | 217 | ||||
| -rw-r--r-- | serialize_lib/init.lua | 74 | ||||
| -rw-r--r-- | serialize_lib/mod.conf | 1 | ||||
| -rw-r--r-- | serialize_lib/readme.md | 7 | ||||
| -rw-r--r-- | serialize_lib/serialize.lua | 269 | ||||
| -rw-r--r-- | serialize_lib/settingtypes.txt | 12 | ||||
| -rw-r--r-- | serialize_lib/tests/serialize_spec.lua | 123 |
7 files changed, 0 insertions, 703 deletions
diff --git a/serialize_lib/atomic.lua b/serialize_lib/atomic.lua deleted file mode 100644 index 182cf42..0000000 --- a/serialize_lib/atomic.lua +++ /dev/null @@ -1,217 +0,0 @@ --- atomic.lua --- Utilities for transaction-like handling of serialized state files --- Also for multiple files that must be synchronous, as advtrains currently requires. - - --- Managing files and backups --- ========================== - ---[[ -The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within -the write, so we have incomplete data. So, the following scheme is applied: -Unix: -1. writes to <filename>.new -2. moves <filename>.new to <filename>, clobbering previous file -Windows: -1. writes to <filename>.new -2. delete <filename> -3. moves <filename>.new to <filename> - -We count a new version of the state as "committed" after stage 2. - -During loading, we apply the following order of precedence: -1. <filename> -2. <filename>.new (windows only, in case we were interrupted just before 3. when saving) - - -All of these functions return either true on success or nil, error on error. -]]-- - -local ser = serialize_lib.serialize - -local windows_mode = false - --- == local functions == - -local function save_atomic_move_file(filename) - --2. if windows mode, delete main file - if windows_mode then - local delsucc, err = os.remove(filename) - if not delsucc then - serialize_lib.log_warn("Unable to delete old savefile '"..filename.."':") - serialize_lib.log_warn(err) - serialize_lib.log_info("Trying to replace the save file anyway now...") - end - end - - --3. move file - local mvsucc, err = os.rename(filename..".new", filename) - if not mvsucc then - if minetest.settings:get_bool("serialize_lib_no_auto_windows_mode") or windows_mode then - serialize_lib.log_error("Unable to replace save file '"..filename.."':") - serialize_lib.log_error(err) - return nil, err - else - -- enable windows mode and try again - serialize_lib.log_info("Unable to replace save file '"..filename.."' by direct renaming:") - serialize_lib.log_info(err) - serialize_lib.log_info("Enabling Windows mode for atomic saving...") - windows_mode = true - return save_atomic_move_file(filename) - end - end - - return true -end - -local function open_file_and_save_callback(callback, filename) - local file, err = io.open(filename, "w") - if not file then - error("Failed opening file '"..filename.."' for write:\n"..err) - end - - callback(file) - return true -end - -local function open_file_and_load_callback(filename, callback) - local file, err = io.open(filename, "r") - if not file then - error("Failed opening file '"..filename.."' for read:\n"..err) - end - - return callback(file) -end - --- == public functions == - --- Load a saved state (according to comment above) --- if 'callback' is nil: reads serialized table. --- returns the read table, or nil,err on error --- if 'callback' is a function (signature func(file_handle) ): --- Counterpart to save_atomic with function argument. Opens the file and calls callback on it. --- If the callback function throws an error, and strict loading is enabled, that error is propagated. --- The callback's first return value is returned by load_atomic -function serialize_lib.load_atomic(filename, callback) - - local cbfunc = callback or ser.read_from_fd - - -- try <filename> - local file, ret = io.open(filename, "r") - if file then - -- read the file using the callback - local success - success, ret = pcall(cbfunc, file) - if success then - return ret - end - end - - if minetest.settings:get_bool("serialize_lib_strict_loading") then - serialize_lib.save_lock = true - error("Loading data from file '"..filename.."' failed:\n" - ..ret.."\nDisable Strict Loading to ignore.") - end - - serialize_lib.log_warn("Loading data from file '"..filename.."' failed, trying .new fallback:") - serialize_lib.log_warn(ret) - - -- try <filename>.new - file, ret = io.open(filename..".new", "r") - if file then - -- read the file using the callback - local success - success, ret = pcall(cbfunc, file) - if success then - return ret - end - end - - serialize_lib.log_error("Unable to load data from '"..filename..".new':") - serialize_lib.log_error(ret) - serialize_lib.log_error("Note: This message is normal when the mod is loaded the first time on this world.") - - return nil, ret -end - --- Save a file atomically (as described above) --- 'data' is the data to be saved (when a callback is used, this can be nil) --- if 'callback' is nil: --- data must be a table, and is serialized into the file --- if 'callback' is a function (signature func(data, file_handle) ): --- Opens the file and calls callback on it. The 'data' argument is the data passed to save_atomic(). --- If the callback function throws an error, and strict loading is enabled, that error is propagated. --- The callback's first return value is returned by load_atomic --- Important: the callback must close the file in all cases! -function serialize_lib.save_atomic(data, filename, callback, config) - if serialize_lib.save_lock then - serialize_lib.log_warn("Instructed to save '"..filename.."', but save lock is active!") - return nil - end - - local cbfunc = callback or ser.write_to_fd - - local file, ret = io.open(filename..".new", "w") - if file then - -- save the file using the callback - local success - success, ret = pcall(cbfunc, data, file) - if success then - return save_atomic_move_file(filename) - end - end - serialize_lib.log_error("Unable to save data to '"..filename..".new':") - serialize_lib.log_error(ret) - return nil, ret -end - - --- Saves multiple files synchronously. First writes all data to all <filename>.new files, --- then moves all files in quick succession to avoid inconsistent backups. --- parts_table is a table where the keys are used as part of the filename and the values --- are the respective data written to it. --- e.g. if parts_table={foo={...}, bar={...}}, then <filename_prefix>foo and <filename_prefix>bar are written out. --- if 'callbacks_table' is defined, it is consulted for callbacks the same way save_atomic does. --- example: if callbacks_table = {foo = func()...}, then the callback is used during writing of file 'foo' (but not for 'bar') --- Note however that you must at least insert a "true" in the parts_table if you don't use the data argument. --- Important: the callback must close the file in all cases! -function serialize_lib.save_atomic_multiple(parts_table, filename_prefix, callbacks_table, config) - if serialize_lib.save_lock then - serialize_lib.log_warn("Instructed to save '"..filename_prefix.."' (multiple), but save lock is active!") - return nil - end - - for subfile, data in pairs(parts_table) do - local filename = filename_prefix..subfile - local cbfunc = ser.write_to_fd - if callbacks_table and callbacks_table[subfile] then - cbfunc = callbacks_table[subfile] - end - - local success = false - local file, ret = io.open(filename..".new", "w") - if file then - -- save the file using the callback - success, ret = pcall(cbfunc, data, file, config) - end - - if not success then - serialize_lib.log_error("Unable to save data to '"..filename..".new':") - serialize_lib.log_error(ret) - return nil, ret - end - end - - local first_error - for file, _ in pairs(parts_table) do - local filename = filename_prefix..file - local succ, err = save_atomic_move_file(filename) - if not succ and not first_error then - first_error = err - end - end - - return not first_error, first_error -- either true,nil or nil,error -end - - diff --git a/serialize_lib/init.lua b/serialize_lib/init.lua deleted file mode 100644 index 20ffa4d..0000000 --- a/serialize_lib/init.lua +++ /dev/null @@ -1,74 +0,0 @@ --- serialize_lib ---[[ - Copyright (C) 2020 Moritz Blei (orwell96) and contributors - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. -]]-- - -serialize_lib = {} - ---[[ Configuration table -Whenever asked for a "config", the following table structure is expected: -config = { - skip_empty_tables = false -- if true, does not store empty tables - -- On next read, keys that mapped to empty tables resolve to nil - -- Used by: write_table_to_file -} -Not all functions use all of the parameters, so you can simplify your config sometimes -]] - --- log utils --- ========= - - -function serialize_lib.log_error(text) - minetest.log("error", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>")) -end -function serialize_lib.log_warn(text) - minetest.log("warning", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>")) -end -function serialize_lib.log_info(text) - minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>")) -end -function serialize_lib.log_debug(text) - minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?")..") DEBUG: "..(text or "<nil>")) -end - --- basic serialization/deserialization --- =================================== - -local mp = minetest.get_modpath(minetest.get_current_modname()) -serialize_lib.serialize = dofile(mp.."/serialize.lua") -dofile(mp.."/atomic.lua") - -local ser = serialize_lib.serialize - --- Opens the passed filename, and returns deserialized table --- When an error occurs, logs an error and returns false -function serialize_lib.read_table_from_file(filename) - local succ, ret = pcall(ser.read_from_file, filename) - if not succ then - serialize_lib.log_error(ret) - end - return ret -end - --- Writes table into file --- When an error occurs, logs an error and returns false -function serialize_lib.write_table_to_file(root_table, filename) - local succ, ret = pcall(ser.write_to_file, root_table, filename) - if not succ then - serialize_lib.log_error(ret) - end - return ret -end - - diff --git a/serialize_lib/mod.conf b/serialize_lib/mod.conf deleted file mode 100644 index ac3a1bd..0000000 --- a/serialize_lib/mod.conf +++ /dev/null @@ -1 +0,0 @@ -name = serialize_lib diff --git a/serialize_lib/readme.md b/serialize_lib/readme.md deleted file mode 100644 index ac364b8..0000000 --- a/serialize_lib/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -# serialize_lib -A Minetest mod library for safely storing large amounts of data in on-disk files. -Created out of the need to have a robust data store for advtrains. - -The main purpose is to load and store large Lua table structures into files, without loading everything in memory and exhausting the function constant limit of LuaJIT. - -Also contains various utilities to handle files on disk in a safe manner, retain multiple versions of the same file a.s.o.
\ No newline at end of file diff --git a/serialize_lib/serialize.lua b/serialize_lib/serialize.lua deleted file mode 100644 index 12a26c4..0000000 --- a/serialize_lib/serialize.lua +++ /dev/null @@ -1,269 +0,0 @@ --- serialize.lua --- Lua-conformant library file that has no minetest dependencies --- Contains the serialization and deserialization routines - ---[[ - -Structure of entry: -[keytype][key]:[valuetype][val] -Types: - B - bool - -> 0=false, 1=true - S - string - -> see below - N - number - -> thing compatible with tonumber() -Table: -[keytype][key]:T -... content is nested in table until the matching -E - -example: -LUA_SER v=1 { -Skey:Svalue key = "value", -N1:Seins [1] = "eins", -B1:T [true] = { -Sa:Sb a = "b", -Sc:B0 c = false, -E } -E } - -String representations: -In strings the following characters are escaped by & -'&' -> '&&' -(line break) -> '&n' -(CR) -> '&r' --> required? -':' -> '&:' -All other characters are unchanged as they bear no special meaning. -]] - -local write_table, literal_to_string, escape_chars, table_is_empty - -function table_is_empty(t) - for _,_ in pairs(t) do - return false - end - return true -end - -function write_table(t, file, config) - local ks, vs, writeit, istable - for key, value in pairs(t) do - ks = value_to_string(key, false) - writeit = true - istable = type(value)=="table" - - if istable then - vs = "T" - if config and config.skip_empty_tables then - writeit = not table_is_empty(value) - end - else - vs = value_to_string(value, true) - end - - if writeit then - file:write(ks..":"..vs.."\n") - - if istable then - write_table(value, file, config) - file:write("E\n") - end - end - end -end - -function value_to_string(t) - if type(t)=="table" then - file:close() - error("Can not serialize a table in the key position!") - elseif type(t)=="boolean" then - if t then - return "B1" - else - return "B0" - end - elseif type(t)=="number" then - return "N"..t - elseif type(t)=="string" then - return "S"..escape_chars(t) - else - --error("Can not serialize '"..type(t).."' type!") - return "S<function>" - end - return str -end - -function escape_chars(str) - local rstr = string.gsub(str, "&", "&&") - rstr = string.gsub(rstr, ":", "&:") - rstr = string.gsub(rstr, "\n", "&n") - return rstr -end - ------- - -local read_table, string_to_value, unescape_chars - -function read_table(t, file) - local line, ks, vs, kv, vv, vt - while true do - line = file:read("*l") - if not line then - file:close() - error("Unexpected EOF or read error!") - end - - if line=="E" then - -- done with this table - return - end - ks, vs = string.match(line, "^(.*[^&]):(.+)$") - if not ks or not vs then - file:close() - error("Unable to parse line: '"..line.."'!") - end - kv = string_to_value(ks) - vv, vt = string_to_value(vs, true) - if vt then - read_table(vv, file) - end - -- put read value in table - t[kv] = vv - end -end - --- returns: value, is_table -function string_to_value(str, table_allow) - local first = string.sub(str, 1,1) - local rest = string.sub(str, 2) - if first=="T" then - if table_allow then - return {}, true - else - file:close() - error("Table not allowed in key component!") - end - elseif first=="N" then - local num = tonumber(rest) - if num then - return num - else - file:close() - error("Unable to parse number: '"..rest.."'!") - end - elseif first=="B" then - if rest=="0" then - return false - elseif rest=="1" then - return true - else - file:close() - error("Unable to parse boolean: '"..rest.."'!") - end - elseif first=="S" then - return unescape_chars(rest) - else - file:close() - error("Unknown literal type '"..first.."' for literal '"..str.."'!") - end -end - -function unescape_chars(str) --TODO - local rstr = string.gsub(str, "&:", ":") - rstr = string.gsub(rstr, "&n", "\n") - rstr = string.gsub(rstr, "&&", "&") - return rstr -end - ------- - ---[[ -config = { - skip_empty_tables = false -- if true, does not store empty tables - -- On next read, keys that mapped to empty tables resolve to nil -} -]] - --- Writes the passed table into the passed file descriptor, and closes the file -local function write_to_fd(root_table, file, config) - file:write("LUA_SER v=1\n") - write_table(root_table, file, config) - file:write("E\nEND_SER\n") - file:close() -end - --- Reads the file contents from the passed file descriptor and returns the table on success --- Throws errors when something is wrong. Closes the file. --- config: see above -local function read_from_fd(file) - local first_line = file:read("*l") - if first_line ~= "LUA_SER v=1" then - file:close() - error("Expected header, got '"..first_line.."' instead!") - end - local t = {} - read_table(t, file) - local last_line = file:read("*l") - file:close() - if last_line ~= "END_SER" then - error("Missing END_SER, got '"..last_line.."' instead!") - end - return t -end - --- Opens the passed filename and serializes root_table into it --- config: see above -function write_to_file(root_table, filename, config) - -- try opening the file - local file, err = io.open(filename, "w") - if not file then - error("Failed opening file '"..filename.."' for write:\n"..err) - end - - write_to_fd(root_table, file, config) - return true -end - --- Opens the passed filename, and returns its deserialized contents -function read_from_file(filename) - -- try opening the file - local file, err = io.open(filename, "r") - if not file then - error("Failed opening file '"..filename.."' for read:\n"..err) - end - - return read_from_fd(file) -end - ---[[ simple unit test -local testtable = { - key = "value", - [1] = "eins", - [true] = { - a = "b", - c = false, - }, - ["es:cape1"] = "foo:bar", - ["es&ca\npe2"] = "baz&bam\nbim", - ["es&&ca&\npe3"] = "baz&&bam&\nbim", - ["es&:cape4"] = "foo\n:bar" -} -local config = {} ---write_to_file(testtable, "test_out", config) -local t = read_from_file("test_out") -write_to_file(t, "test_out_2", config) -local t2 = read_from_file("test_out_2") -write_to_file(t2, "test_out_3", config) - --- test_out_2 and test_out_3 should be equal - ---]] - - -return { - read_from_fd = read_from_fd, - write_to_fd = write_to_fd, - read_from_file = read_from_file, - write_to_file = write_to_file, -} diff --git a/serialize_lib/settingtypes.txt b/serialize_lib/settingtypes.txt deleted file mode 100644 index 3a565a6..0000000 --- a/serialize_lib/settingtypes.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Enable strict file loading mode -# If enabled, if any error occurs during loading of a file using the 'atomic' API, -# an error is thrown. You probably need to disable this option for initial loading after -# creating the world. -serialize_lib_strict_loading (Strict loading) bool false - -# Do not automatically switch to "Windows mode" when saving atomically -# Normally, when renaming <filename>.new to <filename> fails, serialize_lib automatically -# switches to a mode where it deletes <filename> prior to moving. Enable this option to prevent -# this behavior and abort saving instead. -serialize_lib_no_auto_windows_mode (No automatic Windows Mode) bool false - diff --git a/serialize_lib/tests/serialize_spec.lua b/serialize_lib/tests/serialize_spec.lua deleted file mode 100644 index ccc3a67..0000000 --- a/serialize_lib/tests/serialize_spec.lua +++ /dev/null @@ -1,123 +0,0 @@ --- test the serialization function - - -package.path = "../?.lua;" .. package.path - - -ser = require("serialize") - - -local mock_file = {} -_G.mock_file = mock_file -function mock_file:read(arg) - if arg == "*l" then - local l = self.lines[self.pointer or 1] - self.pointer = (self.pointer or 1) + 1 - return l - end -end - -function mock_file:close() - return nil -end - -function mock_file:write(text) - self.content = self.content..text -end - -function mock_file:create(lines) - local f = {} - setmetatable(f, mock_file) - f.lines = lines or {} - f.write = self.write - f.close = self.close - f.read = self.read - f.content = "" - return f -end - - -local testtable = { - key = "value", - [1] = "eins", - [true] = { - a = "b", - c = false, - }, - ["es:cape1"] = "foo:bar", - ["es&ca\npe2"] = "baz&bam\nbim", - ["es&&ca&\npe3"] = "baz&&bam&\nbim", - ["es&:cape4"] = "foo\n:bar" -} -local testser = [[LUA_SER v=1 -B1:T -Sa:Sb -Sc:B0 -E -Skey:Svalue -Ses&&&&ca&&&npe3:Sbaz&&&&bam&&&nbim -N1:Seins -Ses&&&:cape4:Sfoo&n&:bar -Ses&&ca&npe2:Sbaz&&bam&nbim -Ses&:cape1:Sfoo&:bar -E -END_SER -]] - -local function check_write(tb, conf) - f = mock_file:create() - ser.write_to_fd(tb, f, conf or {}) - return f.content -end - -function string:split() - local fields = {} - self:gsub("[^\n]+", function(c) fields[#fields+1] = c end) - return fields -end - -local function check_read(text) - f = mock_file:create(text:split()) - return ser.read_from_fd(f) -end - -local noskip = [[LUA_SER v=1 -N1:T -E -E -END_SER -]] -local skip = [[LUA_SER v=1 -E -END_SER -]] - -describe("write_to_fd", function() - it("serializes a table correctly", function() - assert.equals(check_write(testtable), testser) - end) - it("does not skip empty tables", function() - assert.equals(check_write({{}}),noskip) - end) - it("skips empty tables when needed", function() - - assert.equals(check_write({{}},{skip_empty_tables=true}),skip) - end) -end) - -describe("read_from_fd", function () - it("reads a table correctly", function() - assert.same(check_read(testser),testtable) - end) - it("handles some edge cases correctly", function() - assert.same(check_read(noskip), {{}}) - assert.same(check_read(skip), {}) - end) - it("Read back table", function() - local tb = {} - for k=1,262 do - tb[k] = { "Foo", "bar", k} - end - assert.same(check_read(check_write(tb)), tb) - end) -end) |