diff options
| -rw-r--r-- | serialize_lib/atomic.lua | 215 | ||||
| -rw-r--r-- | serialize_lib/init.lua | 50 | ||||
| -rw-r--r-- | serialize_lib/serialize.lua | 20 | ||||
| -rw-r--r-- | serialize_lib/settingtypes.txt | 12 |
4 files changed, 257 insertions, 40 deletions
diff --git a/serialize_lib/atomic.lua b/serialize_lib/atomic.lua new file mode 100644 index 0000000..85937db --- /dev/null +++ b/serialize_lib/atomic.lua @@ -0,0 +1,215 @@ +-- atomic.lua +-- Utilities for transaction-like handling of serialized state files +-- Also for multiple files that must be synchronous, as advtrains currently requires. + + +-- Managing files and backups +-- ========================== + +--[[ +The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within +the write, so we have incomplete data. So, the following scheme is applied: +Unix: +1. writes to <filename>.new +2. moves <filename>.new to <filename>, clobbering previous file +Windows: +1. writes to <filename>.new +2. delete <filename> +3. moves <filename>.new to <filename> + +We count a new version of the state as "committed" after stage 2. + +During loading, we apply the following order of precedence: +1. <filename> +2. <filename>.new (windows only, in case we were interrupted just before 3. when saving) + + +All of these functions return either true on success or nil, error on error. +]]-- + +local ser = serialize_lib.serialize + +local windows_mode = false + +-- == local functions == + +local function save_atomic_move_file(filename) + --2. if windows mode, delete main file + if windows_mode then + local delsucc, err = os.remove(filename) + if not delsucc then + serialize_lib.log_error("Unable to delete old savefile '"..filename.."':") + serialize_lib.log_error(err) + return nil, err + end + end + + --3. move file + local mvsucc, err = os.rename(filename..".new", filename) + if not mvsucc then + if minetest.settings:get_bool("serialize_lib_no_auto_windows_mode") or windows_mode then + serialize_lib.log_error("Unable to move '"..filename.."':") + serialize_lib.log_error(err) + return nil, err + else + -- enable windows mode and try again + serialize_lib.log_info("Enabling Windows mode for atomic saving...") + windows_mode = true + return save_atomic_move_file(filename) + end + end + + return true +end + +local function open_file_and_save_callback(callback, filename) + local file, err = io.open(filename, "w") + if not file then + error("Failed opening file '"..filename.."' for write:\n"..err) + end + + callback(file) + return true +end + +local function open_file_and_load_callback(filename, callback) + local file, err = io.open(filename, "r") + if not file then + error("Failed opening file '"..filename.."' for read:\n"..err) + end + + return callback(file) +end + +-- == public functions == + +-- Load a saved state (according to comment above) +-- if 'callback' is nil: reads serialized table. +-- returns the read table, or nil,err on error +-- if 'callback' is a function (signature func(file_handle) ): +-- Counterpart to save_atomic with function argument. Opens the file and calls callback on it. +-- If the callback function throws an error, and strict loading is enabled, that error is propagated. +-- The callback's first return value is returned by load_atomic +function serialize_lib.load_atomic(filename, callback) + + local cbfunc = callback or ser.read_from_fd + + -- try <filename> + local file, ret = io.open(filename, "r") + if file then + -- read the file using the callback + local success + success, ret = pcall(cbfunc, file) + if success then + return ret + end + end + + if minetest.settings:get_bool("serialize_lib_strict_loading") then + serialize_lib.save_lock = true + error("Loading data from file '"..filename.."' failed:\n" + ..ret.."\nDisable Strict Loading to ignore.") + end + + serialize_lib.log_warn("Loading data from file '"..filename.."' failed, trying .new fallback:") + serialize_lib.log_warn(ret) + + -- try <filename>.new + file, ret = io.open(filename..".new", "r") + if file then + -- read the file using the callback + local success + success, ret = pcall(cbfunc, file) + if success then + return ret + end + end + + serialize_lib.log_error("Unable to load data from '"..filename..".new':") + serialize_lib.log_error(ret) + serialize_lib.log_error("Note: This message is normal when the mod is loaded the first time on this world.") + + return nil, ret +end + +-- Save a file atomically (as described above) +-- 'data' is the data to be saved (when a callback is used, this can be nil) +-- if 'callback' is nil: +-- data must be a table, and is serialized into the file +-- if 'callback' is a function (signature func(data, file_handle) ): +-- Opens the file and calls callback on it. The 'data' argument is the data passed to save_atomic(). +-- If the callback function throws an error, and strict loading is enabled, that error is propagated. +-- The callback's first return value is returned by load_atomic +-- Important: the callback must close the file in all cases! +function serialize_lib.save_atomic(data, filename, callback, config) + if serialize_lib.save_lock then + serialize_lib.log_warn("Instructed to save '"..filename.."', but save lock is active!") + return nil + end + + local cbfunc = callback or ser.write_to_fd + + local file, ret = io.open(filename..".new", "w") + if file then + -- save the file using the callback + local success + success, ret = pcall(cbfunc, data, file) + if success then + return save_atomic_move_file(filename) + end + end + serialize_lib.log_error("Unable to save data to '"..filename..".new':") + serialize_lib.log_error(ret) + return nil, ret +end + + +-- Saves multiple files synchronously. First writes all data to all <filename>.new files, +-- then moves all files in quick succession to avoid inconsistent backups. +-- parts_table is a table where the keys are used as part of the filename and the values +-- are the respective data written to it. +-- e.g. if parts_table={foo={...}, bar={...}}, then <filename_prefix>foo and <filename_prefix>bar are written out. +-- if 'callbacks_table' is defined, it is consulted for callbacks the same way save_atomic does. +-- example: if callbacks_table = {foo = func()...}, then the callback is used during writing of file 'foo' (but not for 'bar') +-- Note however that you must at least insert a "true" in the parts_table if you don't use the data argument. +-- Important: the callback must close the file in all cases! +function serialize_lib.save_atomic_multiple(parts_table, filename_prefix, callbacks_table, config) + if serialize_lib.save_lock then + serialize_lib.log_warn("Instructed to save '"..filename_prefix.."' (multiple), but save lock is active!") + return nil + end + + for subfile, data in pairs(parts_table) do + local filename = filename_prefix..subfile + local cbfunc = ser.write_to_fd + if callbacks_table and callbacks_table[subfile] then + cbfunc = callbacks_table[subfile] + end + + local success = false + local file, ret = io.open(filename..".new", "w") + if file then + -- save the file using the callback + success, ret = pcall(cbfunc, data, file, config) + end + + if not success then + serialize_lib.log_error("Unable to save data to '"..filename..".new':") + serialize_lib.log_error(ret) + return nil, ret + end + end + + local first_error + for file, _ in pairs(parts_table) do + local filename = filename_prefix..file + local succ, err = save_atomic_move_file(filename) + if not succ and not first_error then + first_error = err + end + end + + return not first_error, first_error -- either true,nil or nil,error +end + + diff --git a/serialize_lib/init.lua b/serialize_lib/init.lua index 7a1a10b..20ffa4d 100644 --- a/serialize_lib/init.lua +++ b/serialize_lib/init.lua @@ -30,63 +30,45 @@ Not all functions use all of the parameters, so you can simplify your config som function serialize_lib.log_error(text) - minetest.log("error", "[serialize_lib] "..text) + minetest.log("error", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>")) end function serialize_lib.log_warn(text) - minetest.log("warning", "[serialize_lib] "..text) + minetest.log("warning", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>")) end function serialize_lib.log_info(text) - minetest.log("action", "[serialize_lib] "..text) + minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>")) end function serialize_lib.log_debug(text) - minetest.log("action", "[serialize_lib](debug) "..text) + minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?")..") DEBUG: "..(text or "<nil>")) end -- basic serialization/deserialization -- =================================== -local ser = dofile("serialize.lua") +local mp = minetest.get_modpath(minetest.get_current_modname()) +serialize_lib.serialize = dofile(mp.."/serialize.lua") +dofile(mp.."/atomic.lua") + +local ser = serialize_lib.serialize -- Opens the passed filename, and returns deserialized table -- When an error occurs, logs an error and returns false function serialize_lib.read_table_from_file(filename) - local succ, err = pcall(ser.read_from_file, filename) + local succ, ret = pcall(ser.read_from_file, filename) if not succ then - serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err) + serialize_lib.log_error(ret) end - return succ + return ret end -- Writes table into file -- When an error occurs, logs an error and returns false -function serialize_lib.write_table_to_file(filename) - local succ, err = pcall(ser.write_to_file, filename) +function serialize_lib.write_table_to_file(root_table, filename) + local succ, ret = pcall(ser.write_to_file, root_table, filename) if not succ then - serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err) + serialize_lib.log_error(ret) end - return succ + return ret end --- Managing files and backups --- ========================== - ---[[ -The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within -the write, so we have incomplete data. So, the following scheme is applied: -1. writes to <filename>.new (if .new already exists, try to complete the moving first) -2. moves <filename> to <filename>.old, possibly overwriting an existing file (special windows behavior) -3. moves <filename>.new to <filename> - -During loading, we apply the following order of precedence: -1. <filename>.new -2. <filename> -3. <filename>.old - -Normal case: <filename> and <filename>.old exist, loading <filename> -Interrupted during write: .new is damaged, loads last regular state -Interrupted during the move operations: either <filename>.new or <filename> represents the latest state -Other corruption: at least the .old state may still be present - -]]-- - diff --git a/serialize_lib/serialize.lua b/serialize_lib/serialize.lua index 8ffd917..2d7c3a0 100644 --- a/serialize_lib/serialize.lua +++ b/serialize_lib/serialize.lua @@ -55,7 +55,7 @@ function write_table(t, file, config) if istable then vs = "T" - if config.skip_empty_tables then + if config and config.skip_empty_tables then writeit = not table_is_empty(value) end else @@ -75,6 +75,7 @@ end function value_to_string(t) if type(t)=="table" then + file:close() error("Can not serialize a table in the key position!") elseif type(t)=="boolean" then if t then @@ -87,6 +88,7 @@ function value_to_string(t) elseif type(t)=="string" then return "S"..escape_chars(t) else + file:close() error("Can not serialize '"..type(t).."' type!") end return str @@ -108,6 +110,7 @@ function read_table(t, file) while true do line = file:read("*l") if not line then + file:close() error("Unexpected EOF or read error!") end @@ -117,6 +120,7 @@ function read_table(t, file) end ks, vs = string.match(line, "^(.+[^&]):(.+)$") if not ks or not vs then + file:close() error("Unable to parse line: '"..line.."'!") end kv = string_to_value(ks) @@ -137,6 +141,7 @@ function string_to_value(str, table_allow) if table_allow then return {}, true else + file:close() error("Table not allowed in key component!") end elseif first=="N" then @@ -144,6 +149,7 @@ function string_to_value(str, table_allow) if num then return num else + file:close() error("Unable to parse number: '"..rest.."'!") end elseif first=="B" then @@ -152,11 +158,13 @@ function string_to_value(str, table_allow) elseif rest=="1" then return true else + file:close() error("Unable to parse boolean: '"..rest.."'!") end elseif first=="S" then return unescape_chars(rest) else + file:close() error("Unknown literal type '"..first.."' for literal '"..str.."'!") end end @@ -177,20 +185,20 @@ config = { } ]] --- Writes the passed table into the passed file descriptor, and closes the file afterwards +-- Writes the passed table into the passed file descriptor, and closes the file local function write_to_fd(root_table, file, config) file:write("LUA_SER v=1\n") write_table(root_table, file, config) file:write("E\nEND_SER\n") - file:close() end -- Reads the file contents from the passed file descriptor and returns the table on success --- Throws errors when something is wrong. +-- Throws errors when something is wrong. Closes the file. -- config: see above local function read_from_fd(file) local first_line = file:read("*l") if first_line ~= "LUA_SER v=1" then + file:close() error("Expected header, got '"..first_line.."' instead!") end local t = {} @@ -209,7 +217,7 @@ function write_to_file(root_table, filename, config) -- try opening the file local file, err = io.open(filename, "w") if not file then - error("Failed opening file '"..filename.."' for write: "..err) + error("Failed opening file '"..filename.."' for write:\n"..err) end write_to_fd(root_table, file, config) @@ -221,7 +229,7 @@ function read_from_file(filename) -- try opening the file local file, err = io.open(filename, "r") if not file then - error("Failed opening file '"..filename.."' for read: "..err) + error("Failed opening file '"..filename.."' for read:\n"..err) end return read_from_fd(file) diff --git a/serialize_lib/settingtypes.txt b/serialize_lib/settingtypes.txt new file mode 100644 index 0000000..3a565a6 --- /dev/null +++ b/serialize_lib/settingtypes.txt @@ -0,0 +1,12 @@ +# Enable strict file loading mode +# If enabled, if any error occurs during loading of a file using the 'atomic' API, +# an error is thrown. You probably need to disable this option for initial loading after +# creating the world. +serialize_lib_strict_loading (Strict loading) bool false + +# Do not automatically switch to "Windows mode" when saving atomically +# Normally, when renaming <filename>.new to <filename> fails, serialize_lib automatically +# switches to a mode where it deletes <filename> prior to moving. Enable this option to prevent +# this behavior and abort saving instead. +serialize_lib_no_auto_windows_mode (No automatic Windows Mode) bool false + |