From 544f8babb5421ed181371c3fcb49478571fab9f4 Mon Sep 17 00:00:00 2001 From: orwell96 Date: Wed, 16 Dec 2020 16:33:41 +0100 Subject: Implement basic serialization and file opening --- init.lua | 92 +++++++++++++++++++++ mod.conf | 1 + readme.md | 7 ++ serialize.lua | 260 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 360 insertions(+) create mode 100644 init.lua create mode 100644 mod.conf create mode 100644 readme.md create mode 100644 serialize.lua diff --git a/init.lua b/init.lua new file mode 100644 index 0000000..7a1a10b --- /dev/null +++ b/init.lua @@ -0,0 +1,92 @@ +-- serialize_lib +--[[ + Copyright (C) 2020 Moritz Blei (orwell96) and contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. +]]-- + +serialize_lib = {} + +--[[ Configuration table +Whenever asked for a "config", the following table structure is expected: +config = { + skip_empty_tables = false -- if true, does not store empty tables + -- On next read, keys that mapped to empty tables resolve to nil + -- Used by: write_table_to_file +} +Not all functions use all of the parameters, so you can simplify your config sometimes +]] + +-- log utils +-- ========= + + +function serialize_lib.log_error(text) + minetest.log("error", "[serialize_lib] "..text) +end +function serialize_lib.log_warn(text) + minetest.log("warning", "[serialize_lib] "..text) +end +function serialize_lib.log_info(text) + minetest.log("action", "[serialize_lib] "..text) +end +function serialize_lib.log_debug(text) + minetest.log("action", "[serialize_lib](debug) "..text) +end + +-- basic serialization/deserialization +-- =================================== + +local ser = dofile("serialize.lua") + +-- Opens the passed filename, and returns deserialized table +-- When an error occurs, logs an error and returns false +function serialize_lib.read_table_from_file(filename) + local succ, err = pcall(ser.read_from_file, filename) + if not succ then + serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err) + end + return succ +end + +-- Writes table into file +-- When an error occurs, logs an error and returns false +function serialize_lib.write_table_to_file(filename) + local succ, err = pcall(ser.write_to_file, filename) + if not succ then + serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err) + end + return succ +end + +-- Managing files and backups +-- ========================== + +--[[ +The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within +the write, so we have incomplete data. So, the following scheme is applied: +1. writes to .new (if .new already exists, try to complete the moving first) +2. moves to .old, possibly overwriting an existing file (special windows behavior) +3. moves .new to + +During loading, we apply the following order of precedence: +1. .new +2. +3. .old + +Normal case: and .old exist, loading +Interrupted during write: .new is damaged, loads last regular state +Interrupted during the move operations: either .new or represents the latest state +Other corruption: at least the .old state may still be present + +]]-- + + diff --git a/mod.conf b/mod.conf new file mode 100644 index 0000000..ac3a1bd --- /dev/null +++ b/mod.conf @@ -0,0 +1 @@ +name = serialize_lib diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..ac364b8 --- /dev/null +++ b/readme.md @@ -0,0 +1,7 @@ +# serialize_lib +A Minetest mod library for safely storing large amounts of data in on-disk files. +Created out of the need to have a robust data store for advtrains. + +The main purpose is to load and store large Lua table structures into files, without loading everything in memory and exhausting the function constant limit of LuaJIT. + +Also contains various utilities to handle files on disk in a safe manner, retain multiple versions of the same file a.s.o. \ No newline at end of file diff --git a/serialize.lua b/serialize.lua new file mode 100644 index 0000000..8ffd917 --- /dev/null +++ b/serialize.lua @@ -0,0 +1,260 @@ +-- serialize.lua +-- Lua-conformant library file that has no minetest dependencies +-- Contains the serialization and deserialization routines + +--[[ + +Structure of entry: +[keytype][key]:[valuetype][val] +Types: + B - bool + -> 0=false, 1=true + S - string + -> see below + N - number + -> thing compatible with tonumber() +Table: +[keytype][key]:T +... content is nested in table until the matching +E + +example: +LUA_SER v=1 { +Skey:Svalue key = "value", +N1:Seins [1] = "eins", +B1:T [true] = { +Sa:Sb a = "b", +Sc:B0 c = false, +E } +E } + +String representations: +In strings the following characters are escaped by & +'&' -> '&&' +(line break) -> '&n' +(CR) -> '&r' --> required? +':' -> '&:' +All other characters are unchanged as they bear no special meaning. +]] + +local write_table, literal_to_string, escape_chars, table_is_empty + +function table_is_empty(t) + for _,_ in pairs(t) do + return false + end + return true +end + +function write_table(t, file, config) + local ks, vs, writeit, istable + for key, value in pairs(t) do + ks = value_to_string(key, false) + writeit = true + istable = type(value)=="table" + + if istable then + vs = "T" + if config.skip_empty_tables then + writeit = not table_is_empty(value) + end + else + vs = value_to_string(value, true) + end + + if writeit then + file:write(ks..":"..vs.."\n") + + if istable then + write_table(value, file, config) + file:write("E\n") + end + end + end +end + +function value_to_string(t) + if type(t)=="table" then + error("Can not serialize a table in the key position!") + elseif type(t)=="boolean" then + if t then + return "B1" + else + return "B0" + end + elseif type(t)=="number" then + return "N"..t + elseif type(t)=="string" then + return "S"..escape_chars(t) + else + error("Can not serialize '"..type(t).."' type!") + end + return str +end + +function escape_chars(str) + local rstr = string.gsub(str, "&", "&&") + rstr = string.gsub(rstr, ":", "&:") + rstr = string.gsub(rstr, "\n", "&n") + return rstr +end + +------ + +local read_table, string_to_value, unescape_chars + +function read_table(t, file) + local line, ks, vs, kv, vv, vt + while true do + line = file:read("*l") + if not line then + error("Unexpected EOF or read error!") + end + + if line=="E" then + -- done with this table + return + end + ks, vs = string.match(line, "^(.+[^&]):(.+)$") + if not ks or not vs then + error("Unable to parse line: '"..line.."'!") + end + kv = string_to_value(ks) + vv, vt = string_to_value(vs, true) + if vt then + read_table(vv, file) + end + -- put read value in table + t[kv] = vv + end +end + +-- returns: value, is_table +function string_to_value(str, table_allow) + local first = string.sub(str, 1,1) + local rest = string.sub(str, 2) + if first=="T" then + if table_allow then + return {}, true + else + error("Table not allowed in key component!") + end + elseif first=="N" then + local num = tonumber(rest) + if num then + return num + else + error("Unable to parse number: '"..rest.."'!") + end + elseif first=="B" then + if rest=="0" then + return false + elseif rest=="1" then + return true + else + error("Unable to parse boolean: '"..rest.."'!") + end + elseif first=="S" then + return unescape_chars(rest) + else + error("Unknown literal type '"..first.."' for literal '"..str.."'!") + end +end + +function unescape_chars(str) --TODO + local rstr = string.gsub(str, "&:", ":") + rstr = string.gsub(rstr, "&n", "\n") + rstr = string.gsub(rstr, "&&", "&") + return rstr +end + +------ + +--[[ +config = { + skip_empty_tables = false -- if true, does not store empty tables + -- On next read, keys that mapped to empty tables resolve to nil +} +]] + +-- Writes the passed table into the passed file descriptor, and closes the file afterwards +local function write_to_fd(root_table, file, config) + file:write("LUA_SER v=1\n") + write_table(root_table, file, config) + file:write("E\nEND_SER\n") + file:close() +end + +-- Reads the file contents from the passed file descriptor and returns the table on success +-- Throws errors when something is wrong. +-- config: see above +local function read_from_fd(file) + local first_line = file:read("*l") + if first_line ~= "LUA_SER v=1" then + error("Expected header, got '"..first_line.."' instead!") + end + local t = {} + read_table(t, file) + local last_line = file:read("*l") + file:close() + if last_line ~= "END_SER" then + error("Missing END_SER, got '"..last_line.."' instead!") + end + return t +end + +-- Opens the passed filename and serializes root_table into it +-- config: see above +function write_to_file(root_table, filename, config) + -- try opening the file + local file, err = io.open(filename, "w") + if not file then + error("Failed opening file '"..filename.."' for write: "..err) + end + + write_to_fd(root_table, file, config) + return true +end + +-- Opens the passed filename, and returns its deserialized contents +function read_from_file(filename) + -- try opening the file + local file, err = io.open(filename, "r") + if not file then + error("Failed opening file '"..filename.."' for read: "..err) + end + + return read_from_fd(file) +end + +--[[ simple unit test +local testtable = { + key = "value", + [1] = "eins", + [true] = { + a = "b", + c = false, + }, + ["es:cape1"] = "foo:bar", + ["es&ca\npe2"] = "baz&bam\nbim", + ["es&&ca&\npe3"] = "baz&&bam&\nbim", + ["es&:cape4"] = "foo\n:bar" +} +local config = {} +--write_to_file(testtable, "test_out", config) +local t = read_from_file("test_out") +write_to_file(t, "test_out_2", config) +local t2 = read_from_file("test_out_2") +write_to_file(t2, "test_out_3", config) + +-- test_out_2 and test_out_3 should be equal + +--]] + + +return { + read_from_fd = read_from_fd, + write_to_fd = write_to_fd, + read_from_file = read_from_file, + write_to_file = write_to_file, +} -- cgit v1.2.3 From 02d845e83d7efb84216c7ac746800fd742426e06 Mon Sep 17 00:00:00 2001 From: orwell96 Date: Mon, 21 Dec 2020 20:03:49 +0100 Subject: Serialize_lib: finish up and add atomic api --- atomic.lua | 215 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ init.lua | 50 +++++-------- serialize.lua | 20 ++++-- settingtypes.txt | 12 ++++ 4 files changed, 257 insertions(+), 40 deletions(-) create mode 100644 atomic.lua create mode 100644 settingtypes.txt diff --git a/atomic.lua b/atomic.lua new file mode 100644 index 0000000..85937db --- /dev/null +++ b/atomic.lua @@ -0,0 +1,215 @@ +-- atomic.lua +-- Utilities for transaction-like handling of serialized state files +-- Also for multiple files that must be synchronous, as advtrains currently requires. + + +-- Managing files and backups +-- ========================== + +--[[ +The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within +the write, so we have incomplete data. So, the following scheme is applied: +Unix: +1. writes to .new +2. moves .new to , clobbering previous file +Windows: +1. writes to .new +2. delete +3. moves .new to + +We count a new version of the state as "committed" after stage 2. + +During loading, we apply the following order of precedence: +1. +2. .new (windows only, in case we were interrupted just before 3. when saving) + + +All of these functions return either true on success or nil, error on error. +]]-- + +local ser = serialize_lib.serialize + +local windows_mode = false + +-- == local functions == + +local function save_atomic_move_file(filename) + --2. if windows mode, delete main file + if windows_mode then + local delsucc, err = os.remove(filename) + if not delsucc then + serialize_lib.log_error("Unable to delete old savefile '"..filename.."':") + serialize_lib.log_error(err) + return nil, err + end + end + + --3. move file + local mvsucc, err = os.rename(filename..".new", filename) + if not mvsucc then + if minetest.settings:get_bool("serialize_lib_no_auto_windows_mode") or windows_mode then + serialize_lib.log_error("Unable to move '"..filename.."':") + serialize_lib.log_error(err) + return nil, err + else + -- enable windows mode and try again + serialize_lib.log_info("Enabling Windows mode for atomic saving...") + windows_mode = true + return save_atomic_move_file(filename) + end + end + + return true +end + +local function open_file_and_save_callback(callback, filename) + local file, err = io.open(filename, "w") + if not file then + error("Failed opening file '"..filename.."' for write:\n"..err) + end + + callback(file) + return true +end + +local function open_file_and_load_callback(filename, callback) + local file, err = io.open(filename, "r") + if not file then + error("Failed opening file '"..filename.."' for read:\n"..err) + end + + return callback(file) +end + +-- == public functions == + +-- Load a saved state (according to comment above) +-- if 'callback' is nil: reads serialized table. +-- returns the read table, or nil,err on error +-- if 'callback' is a function (signature func(file_handle) ): +-- Counterpart to save_atomic with function argument. Opens the file and calls callback on it. +-- If the callback function throws an error, and strict loading is enabled, that error is propagated. +-- The callback's first return value is returned by load_atomic +function serialize_lib.load_atomic(filename, callback) + + local cbfunc = callback or ser.read_from_fd + + -- try + local file, ret = io.open(filename, "r") + if file then + -- read the file using the callback + local success + success, ret = pcall(cbfunc, file) + if success then + return ret + end + end + + if minetest.settings:get_bool("serialize_lib_strict_loading") then + serialize_lib.save_lock = true + error("Loading data from file '"..filename.."' failed:\n" + ..ret.."\nDisable Strict Loading to ignore.") + end + + serialize_lib.log_warn("Loading data from file '"..filename.."' failed, trying .new fallback:") + serialize_lib.log_warn(ret) + + -- try .new + file, ret = io.open(filename..".new", "r") + if file then + -- read the file using the callback + local success + success, ret = pcall(cbfunc, file) + if success then + return ret + end + end + + serialize_lib.log_error("Unable to load data from '"..filename..".new':") + serialize_lib.log_error(ret) + serialize_lib.log_error("Note: This message is normal when the mod is loaded the first time on this world.") + + return nil, ret +end + +-- Save a file atomically (as described above) +-- 'data' is the data to be saved (when a callback is used, this can be nil) +-- if 'callback' is nil: +-- data must be a table, and is serialized into the file +-- if 'callback' is a function (signature func(data, file_handle) ): +-- Opens the file and calls callback on it. The 'data' argument is the data passed to save_atomic(). +-- If the callback function throws an error, and strict loading is enabled, that error is propagated. +-- The callback's first return value is returned by load_atomic +-- Important: the callback must close the file in all cases! +function serialize_lib.save_atomic(data, filename, callback, config) + if serialize_lib.save_lock then + serialize_lib.log_warn("Instructed to save '"..filename.."', but save lock is active!") + return nil + end + + local cbfunc = callback or ser.write_to_fd + + local file, ret = io.open(filename..".new", "w") + if file then + -- save the file using the callback + local success + success, ret = pcall(cbfunc, data, file) + if success then + return save_atomic_move_file(filename) + end + end + serialize_lib.log_error("Unable to save data to '"..filename..".new':") + serialize_lib.log_error(ret) + return nil, ret +end + + +-- Saves multiple files synchronously. First writes all data to all .new files, +-- then moves all files in quick succession to avoid inconsistent backups. +-- parts_table is a table where the keys are used as part of the filename and the values +-- are the respective data written to it. +-- e.g. if parts_table={foo={...}, bar={...}}, then foo and bar are written out. +-- if 'callbacks_table' is defined, it is consulted for callbacks the same way save_atomic does. +-- example: if callbacks_table = {foo = func()...}, then the callback is used during writing of file 'foo' (but not for 'bar') +-- Note however that you must at least insert a "true" in the parts_table if you don't use the data argument. +-- Important: the callback must close the file in all cases! +function serialize_lib.save_atomic_multiple(parts_table, filename_prefix, callbacks_table, config) + if serialize_lib.save_lock then + serialize_lib.log_warn("Instructed to save '"..filename_prefix.."' (multiple), but save lock is active!") + return nil + end + + for subfile, data in pairs(parts_table) do + local filename = filename_prefix..subfile + local cbfunc = ser.write_to_fd + if callbacks_table and callbacks_table[subfile] then + cbfunc = callbacks_table[subfile] + end + + local success = false + local file, ret = io.open(filename..".new", "w") + if file then + -- save the file using the callback + success, ret = pcall(cbfunc, data, file, config) + end + + if not success then + serialize_lib.log_error("Unable to save data to '"..filename..".new':") + serialize_lib.log_error(ret) + return nil, ret + end + end + + local first_error + for file, _ in pairs(parts_table) do + local filename = filename_prefix..file + local succ, err = save_atomic_move_file(filename) + if not succ and not first_error then + first_error = err + end + end + + return not first_error, first_error -- either true,nil or nil,error +end + + diff --git a/init.lua b/init.lua index 7a1a10b..20ffa4d 100644 --- a/init.lua +++ b/init.lua @@ -30,63 +30,45 @@ Not all functions use all of the parameters, so you can simplify your config som function serialize_lib.log_error(text) - minetest.log("error", "[serialize_lib] "..text) + minetest.log("error", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "")) end function serialize_lib.log_warn(text) - minetest.log("warning", "[serialize_lib] "..text) + minetest.log("warning", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "")) end function serialize_lib.log_info(text) - minetest.log("action", "[serialize_lib] "..text) + minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "")) end function serialize_lib.log_debug(text) - minetest.log("action", "[serialize_lib](debug) "..text) + minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?")..") DEBUG: "..(text or "")) end -- basic serialization/deserialization -- =================================== -local ser = dofile("serialize.lua") +local mp = minetest.get_modpath(minetest.get_current_modname()) +serialize_lib.serialize = dofile(mp.."/serialize.lua") +dofile(mp.."/atomic.lua") + +local ser = serialize_lib.serialize -- Opens the passed filename, and returns deserialized table -- When an error occurs, logs an error and returns false function serialize_lib.read_table_from_file(filename) - local succ, err = pcall(ser.read_from_file, filename) + local succ, ret = pcall(ser.read_from_file, filename) if not succ then - serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err) + serialize_lib.log_error(ret) end - return succ + return ret end -- Writes table into file -- When an error occurs, logs an error and returns false -function serialize_lib.write_table_to_file(filename) - local succ, err = pcall(ser.write_to_file, filename) +function serialize_lib.write_table_to_file(root_table, filename) + local succ, ret = pcall(ser.write_to_file, root_table, filename) if not succ then - serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err) + serialize_lib.log_error(ret) end - return succ + return ret end --- Managing files and backups --- ========================== - ---[[ -The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within -the write, so we have incomplete data. So, the following scheme is applied: -1. writes to .new (if .new already exists, try to complete the moving first) -2. moves to .old, possibly overwriting an existing file (special windows behavior) -3. moves .new to - -During loading, we apply the following order of precedence: -1. .new -2. -3. .old - -Normal case: and .old exist, loading -Interrupted during write: .new is damaged, loads last regular state -Interrupted during the move operations: either .new or represents the latest state -Other corruption: at least the .old state may still be present - -]]-- - diff --git a/serialize.lua b/serialize.lua index 8ffd917..2d7c3a0 100644 --- a/serialize.lua +++ b/serialize.lua @@ -55,7 +55,7 @@ function write_table(t, file, config) if istable then vs = "T" - if config.skip_empty_tables then + if config and config.skip_empty_tables then writeit = not table_is_empty(value) end else @@ -75,6 +75,7 @@ end function value_to_string(t) if type(t)=="table" then + file:close() error("Can not serialize a table in the key position!") elseif type(t)=="boolean" then if t then @@ -87,6 +88,7 @@ function value_to_string(t) elseif type(t)=="string" then return "S"..escape_chars(t) else + file:close() error("Can not serialize '"..type(t).."' type!") end return str @@ -108,6 +110,7 @@ function read_table(t, file) while true do line = file:read("*l") if not line then + file:close() error("Unexpected EOF or read error!") end @@ -117,6 +120,7 @@ function read_table(t, file) end ks, vs = string.match(line, "^(.+[^&]):(.+)$") if not ks or not vs then + file:close() error("Unable to parse line: '"..line.."'!") end kv = string_to_value(ks) @@ -137,6 +141,7 @@ function string_to_value(str, table_allow) if table_allow then return {}, true else + file:close() error("Table not allowed in key component!") end elseif first=="N" then @@ -144,6 +149,7 @@ function string_to_value(str, table_allow) if num then return num else + file:close() error("Unable to parse number: '"..rest.."'!") end elseif first=="B" then @@ -152,11 +158,13 @@ function string_to_value(str, table_allow) elseif rest=="1" then return true else + file:close() error("Unable to parse boolean: '"..rest.."'!") end elseif first=="S" then return unescape_chars(rest) else + file:close() error("Unknown literal type '"..first.."' for literal '"..str.."'!") end end @@ -177,20 +185,20 @@ config = { } ]] --- Writes the passed table into the passed file descriptor, and closes the file afterwards +-- Writes the passed table into the passed file descriptor, and closes the file local function write_to_fd(root_table, file, config) file:write("LUA_SER v=1\n") write_table(root_table, file, config) file:write("E\nEND_SER\n") - file:close() end -- Reads the file contents from the passed file descriptor and returns the table on success --- Throws errors when something is wrong. +-- Throws errors when something is wrong. Closes the file. -- config: see above local function read_from_fd(file) local first_line = file:read("*l") if first_line ~= "LUA_SER v=1" then + file:close() error("Expected header, got '"..first_line.."' instead!") end local t = {} @@ -209,7 +217,7 @@ function write_to_file(root_table, filename, config) -- try opening the file local file, err = io.open(filename, "w") if not file then - error("Failed opening file '"..filename.."' for write: "..err) + error("Failed opening file '"..filename.."' for write:\n"..err) end write_to_fd(root_table, file, config) @@ -221,7 +229,7 @@ function read_from_file(filename) -- try opening the file local file, err = io.open(filename, "r") if not file then - error("Failed opening file '"..filename.."' for read: "..err) + error("Failed opening file '"..filename.."' for read:\n"..err) end return read_from_fd(file) diff --git a/settingtypes.txt b/settingtypes.txt new file mode 100644 index 0000000..3a565a6 --- /dev/null +++ b/settingtypes.txt @@ -0,0 +1,12 @@ +# Enable strict file loading mode +# If enabled, if any error occurs during loading of a file using the 'atomic' API, +# an error is thrown. You probably need to disable this option for initial loading after +# creating the world. +serialize_lib_strict_loading (Strict loading) bool false + +# Do not automatically switch to "Windows mode" when saving atomically +# Normally, when renaming .new to fails, serialize_lib automatically +# switches to a mode where it deletes prior to moving. Enable this option to prevent +# this behavior and abort saving instead. +serialize_lib_no_auto_windows_mode (No automatic Windows Mode) bool false + -- cgit v1.2.3 From 19b399a01e451cd3f386bc8353f2d6d902c00d15 Mon Sep 17 00:00:00 2001 From: orwell96 Date: Sat, 2 Jan 2021 21:22:45 +0100 Subject: serialize_lib: Allow empty strings in key --- serialize.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serialize.lua b/serialize.lua index 2d7c3a0..436bf18 100644 --- a/serialize.lua +++ b/serialize.lua @@ -118,7 +118,7 @@ function read_table(t, file) -- done with this table return end - ks, vs = string.match(line, "^(.+[^&]):(.+)$") + ks, vs = string.match(line, "^(.*[^&]):(.+)$") if not ks or not vs then file:close() error("Unable to parse line: '"..line.."'!") -- cgit v1.2.3 From 8ffdd8c293b84ffcf91c457a8accbd3e979eb38e Mon Sep 17 00:00:00 2001 From: orwell96 Date: Sat, 2 Jan 2021 20:57:45 +0100 Subject: Some more serializer fixes (backported from new_lzb): - Move DUMP_DEBUG_SAVE block before the actual saving so it can be used to trace serializer errors - Don't crash on functions in data, ignore them silently - Increase the save interval --- serialize.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serialize.lua b/serialize.lua index 436bf18..a525e9e 100644 --- a/serialize.lua +++ b/serialize.lua @@ -88,8 +88,8 @@ function value_to_string(t) elseif type(t)=="string" then return "S"..escape_chars(t) else - file:close() - error("Can not serialize '"..type(t).."' type!") + --error("Can not serialize '"..type(t).."' type!") + return "S" end return str end -- cgit v1.2.3 From ed165e1b1784f4fed38bb69441051c1ce6b137c8 Mon Sep 17 00:00:00 2001 From: orwell96 Date: Sun, 17 Jan 2021 10:27:16 +0100 Subject: Fix atomic saving on windows when save file does not exist --- atomic.lua | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/atomic.lua b/atomic.lua index 85937db..182cf42 100644 --- a/atomic.lua +++ b/atomic.lua @@ -38,9 +38,9 @@ local function save_atomic_move_file(filename) if windows_mode then local delsucc, err = os.remove(filename) if not delsucc then - serialize_lib.log_error("Unable to delete old savefile '"..filename.."':") - serialize_lib.log_error(err) - return nil, err + serialize_lib.log_warn("Unable to delete old savefile '"..filename.."':") + serialize_lib.log_warn(err) + serialize_lib.log_info("Trying to replace the save file anyway now...") end end @@ -48,11 +48,13 @@ local function save_atomic_move_file(filename) local mvsucc, err = os.rename(filename..".new", filename) if not mvsucc then if minetest.settings:get_bool("serialize_lib_no_auto_windows_mode") or windows_mode then - serialize_lib.log_error("Unable to move '"..filename.."':") + serialize_lib.log_error("Unable to replace save file '"..filename.."':") serialize_lib.log_error(err) return nil, err else -- enable windows mode and try again + serialize_lib.log_info("Unable to replace save file '"..filename.."' by direct renaming:") + serialize_lib.log_info(err) serialize_lib.log_info("Enabling Windows mode for atomic saving...") windows_mode = true return save_atomic_move_file(filename) -- cgit v1.2.3 From cdad238183e4aa62df16b2367321abf0f7a517c8 Mon Sep 17 00:00:00 2001 From: Blockhead Date: Mon, 18 Jan 2021 15:27:50 +0100 Subject: Fix serialisation: breach of contract, file left open Previous commit did not fix saving, but is kept because there is a corner case for which it is required (see MT forum) --- serialize.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/serialize.lua b/serialize.lua index a525e9e..12a26c4 100644 --- a/serialize.lua +++ b/serialize.lua @@ -190,6 +190,7 @@ local function write_to_fd(root_table, file, config) file:write("LUA_SER v=1\n") write_table(root_table, file, config) file:write("E\nEND_SER\n") + file:close() end -- Reads the file contents from the passed file descriptor and returns the table on success -- cgit v1.2.3 From 1b29cf6540b6a500f35abcbf8a65622ba7805e40 Mon Sep 17 00:00:00 2001 From: Gabriel Pérez-Cerezo Date: Sun, 31 Jan 2021 11:53:31 +0100 Subject: add unit tests for serialize_lib --- tests/serialize_spec.lua | 123 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 tests/serialize_spec.lua diff --git a/tests/serialize_spec.lua b/tests/serialize_spec.lua new file mode 100644 index 0000000..ccc3a67 --- /dev/null +++ b/tests/serialize_spec.lua @@ -0,0 +1,123 @@ +-- test the serialization function + + +package.path = "../?.lua;" .. package.path + + +ser = require("serialize") + + +local mock_file = {} +_G.mock_file = mock_file +function mock_file:read(arg) + if arg == "*l" then + local l = self.lines[self.pointer or 1] + self.pointer = (self.pointer or 1) + 1 + return l + end +end + +function mock_file:close() + return nil +end + +function mock_file:write(text) + self.content = self.content..text +end + +function mock_file:create(lines) + local f = {} + setmetatable(f, mock_file) + f.lines = lines or {} + f.write = self.write + f.close = self.close + f.read = self.read + f.content = "" + return f +end + + +local testtable = { + key = "value", + [1] = "eins", + [true] = { + a = "b", + c = false, + }, + ["es:cape1"] = "foo:bar", + ["es&ca\npe2"] = "baz&bam\nbim", + ["es&&ca&\npe3"] = "baz&&bam&\nbim", + ["es&:cape4"] = "foo\n:bar" +} +local testser = [[LUA_SER v=1 +B1:T +Sa:Sb +Sc:B0 +E +Skey:Svalue +Ses&&&&ca&&&npe3:Sbaz&&&&bam&&&nbim +N1:Seins +Ses&&&:cape4:Sfoo&n&:bar +Ses&&ca&npe2:Sbaz&&bam&nbim +Ses&:cape1:Sfoo&:bar +E +END_SER +]] + +local function check_write(tb, conf) + f = mock_file:create() + ser.write_to_fd(tb, f, conf or {}) + return f.content +end + +function string:split() + local fields = {} + self:gsub("[^\n]+", function(c) fields[#fields+1] = c end) + return fields +end + +local function check_read(text) + f = mock_file:create(text:split()) + return ser.read_from_fd(f) +end + +local noskip = [[LUA_SER v=1 +N1:T +E +E +END_SER +]] +local skip = [[LUA_SER v=1 +E +END_SER +]] + +describe("write_to_fd", function() + it("serializes a table correctly", function() + assert.equals(check_write(testtable), testser) + end) + it("does not skip empty tables", function() + assert.equals(check_write({{}}),noskip) + end) + it("skips empty tables when needed", function() + + assert.equals(check_write({{}},{skip_empty_tables=true}),skip) + end) +end) + +describe("read_from_fd", function () + it("reads a table correctly", function() + assert.same(check_read(testser),testtable) + end) + it("handles some edge cases correctly", function() + assert.same(check_read(noskip), {{}}) + assert.same(check_read(skip), {}) + end) + it("Read back table", function() + local tb = {} + for k=1,262 do + tb[k] = { "Foo", "bar", k} + end + assert.same(check_read(check_write(tb)), tb) + end) +end) -- cgit v1.2.3 From a6e8b8b4353863ad563a4d5187f40fea702ea2de Mon Sep 17 00:00:00 2001 From: orwell96 Date: Sat, 13 Mar 2021 11:16:05 +0100 Subject: mod.conf: Add more information about the mod, not just the name --- mod.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mod.conf b/mod.conf index ac3a1bd..a2fd6bb 100644 --- a/mod.conf +++ b/mod.conf @@ -1 +1,4 @@ -name = serialize_lib +name=serialize_lib +title=Lua Serialization Library +description=A library to efficiently write and read Lua tables to/from a file, with support for atomic operations +author=orwell96 -- cgit v1.2.3