aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authororwell96 <orwell@bleipb.de>2020-12-21 20:03:49 +0100
committerorwell96 <orwell@bleipb.de>2021-01-12 13:27:00 +0100
commitd0bd4ac30e8f14d40df01a91e346e7c2c589e0a9 (patch)
treee3d54c790d27a4ed4a686a26b65a8cb7777130a2
parente6b656e93723ea2f0d36c8da39baba4fa099d711 (diff)
downloadadvtrains-d0bd4ac30e8f14d40df01a91e346e7c2c589e0a9.tar.gz
advtrains-d0bd4ac30e8f14d40df01a91e346e7c2c589e0a9.tar.bz2
advtrains-d0bd4ac30e8f14d40df01a91e346e7c2c589e0a9.zip
Serialize_lib: finish up and add atomic api
-rw-r--r--serialize_lib/atomic.lua215
-rw-r--r--serialize_lib/init.lua50
-rw-r--r--serialize_lib/serialize.lua20
-rw-r--r--serialize_lib/settingtypes.txt12
4 files changed, 257 insertions, 40 deletions
diff --git a/serialize_lib/atomic.lua b/serialize_lib/atomic.lua
new file mode 100644
index 0000000..85937db
--- /dev/null
+++ b/serialize_lib/atomic.lua
@@ -0,0 +1,215 @@
+-- atomic.lua
+-- Utilities for transaction-like handling of serialized state files
+-- Also for multiple files that must be synchronous, as advtrains currently requires.
+
+
+-- Managing files and backups
+-- ==========================
+
+--[[
+The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within
+the write, so we have incomplete data. So, the following scheme is applied:
+Unix:
+1. writes to <filename>.new
+2. moves <filename>.new to <filename>, clobbering previous file
+Windows:
+1. writes to <filename>.new
+2. delete <filename>
+3. moves <filename>.new to <filename>
+
+We count a new version of the state as "committed" after stage 2.
+
+During loading, we apply the following order of precedence:
+1. <filename>
+2. <filename>.new (windows only, in case we were interrupted just before 3. when saving)
+
+
+All of these functions return either true on success or nil, error on error.
+]]--
+
+local ser = serialize_lib.serialize
+
+local windows_mode = false
+
+-- == local functions ==
+
+local function save_atomic_move_file(filename)
+ --2. if windows mode, delete main file
+ if windows_mode then
+ local delsucc, err = os.remove(filename)
+ if not delsucc then
+ serialize_lib.log_error("Unable to delete old savefile '"..filename.."':")
+ serialize_lib.log_error(err)
+ return nil, err
+ end
+ end
+
+ --3. move file
+ local mvsucc, err = os.rename(filename..".new", filename)
+ if not mvsucc then
+ if minetest.settings:get_bool("serialize_lib_no_auto_windows_mode") or windows_mode then
+ serialize_lib.log_error("Unable to move '"..filename.."':")
+ serialize_lib.log_error(err)
+ return nil, err
+ else
+ -- enable windows mode and try again
+ serialize_lib.log_info("Enabling Windows mode for atomic saving...")
+ windows_mode = true
+ return save_atomic_move_file(filename)
+ end
+ end
+
+ return true
+end
+
+local function open_file_and_save_callback(callback, filename)
+ local file, err = io.open(filename, "w")
+ if not file then
+ error("Failed opening file '"..filename.."' for write:\n"..err)
+ end
+
+ callback(file)
+ return true
+end
+
+local function open_file_and_load_callback(filename, callback)
+ local file, err = io.open(filename, "r")
+ if not file then
+ error("Failed opening file '"..filename.."' for read:\n"..err)
+ end
+
+ return callback(file)
+end
+
+-- == public functions ==
+
+-- Load a saved state (according to comment above)
+-- if 'callback' is nil: reads serialized table.
+-- returns the read table, or nil,err on error
+-- if 'callback' is a function (signature func(file_handle) ):
+-- Counterpart to save_atomic with function argument. Opens the file and calls callback on it.
+-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
+-- The callback's first return value is returned by load_atomic
+function serialize_lib.load_atomic(filename, callback)
+
+ local cbfunc = callback or ser.read_from_fd
+
+ -- try <filename>
+ local file, ret = io.open(filename, "r")
+ if file then
+ -- read the file using the callback
+ local success
+ success, ret = pcall(cbfunc, file)
+ if success then
+ return ret
+ end
+ end
+
+ if minetest.settings:get_bool("serialize_lib_strict_loading") then
+ serialize_lib.save_lock = true
+ error("Loading data from file '"..filename.."' failed:\n"
+ ..ret.."\nDisable Strict Loading to ignore.")
+ end
+
+ serialize_lib.log_warn("Loading data from file '"..filename.."' failed, trying .new fallback:")
+ serialize_lib.log_warn(ret)
+
+ -- try <filename>.new
+ file, ret = io.open(filename..".new", "r")
+ if file then
+ -- read the file using the callback
+ local success
+ success, ret = pcall(cbfunc, file)
+ if success then
+ return ret
+ end
+ end
+
+ serialize_lib.log_error("Unable to load data from '"..filename..".new':")
+ serialize_lib.log_error(ret)
+ serialize_lib.log_error("Note: This message is normal when the mod is loaded the first time on this world.")
+
+ return nil, ret
+end
+
+-- Save a file atomically (as described above)
+-- 'data' is the data to be saved (when a callback is used, this can be nil)
+-- if 'callback' is nil:
+-- data must be a table, and is serialized into the file
+-- if 'callback' is a function (signature func(data, file_handle) ):
+-- Opens the file and calls callback on it. The 'data' argument is the data passed to save_atomic().
+-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
+-- The callback's first return value is returned by load_atomic
+-- Important: the callback must close the file in all cases!
+function serialize_lib.save_atomic(data, filename, callback, config)
+ if serialize_lib.save_lock then
+ serialize_lib.log_warn("Instructed to save '"..filename.."', but save lock is active!")
+ return nil
+ end
+
+ local cbfunc = callback or ser.write_to_fd
+
+ local file, ret = io.open(filename..".new", "w")
+ if file then
+ -- save the file using the callback
+ local success
+ success, ret = pcall(cbfunc, data, file)
+ if success then
+ return save_atomic_move_file(filename)
+ end
+ end
+ serialize_lib.log_error("Unable to save data to '"..filename..".new':")
+ serialize_lib.log_error(ret)
+ return nil, ret
+end
+
+
+-- Saves multiple files synchronously. First writes all data to all <filename>.new files,
+-- then moves all files in quick succession to avoid inconsistent backups.
+-- parts_table is a table where the keys are used as part of the filename and the values
+-- are the respective data written to it.
+-- e.g. if parts_table={foo={...}, bar={...}}, then <filename_prefix>foo and <filename_prefix>bar are written out.
+-- if 'callbacks_table' is defined, it is consulted for callbacks the same way save_atomic does.
+-- example: if callbacks_table = {foo = func()...}, then the callback is used during writing of file 'foo' (but not for 'bar')
+-- Note however that you must at least insert a "true" in the parts_table if you don't use the data argument.
+-- Important: the callback must close the file in all cases!
+function serialize_lib.save_atomic_multiple(parts_table, filename_prefix, callbacks_table, config)
+ if serialize_lib.save_lock then
+ serialize_lib.log_warn("Instructed to save '"..filename_prefix.."' (multiple), but save lock is active!")
+ return nil
+ end
+
+ for subfile, data in pairs(parts_table) do
+ local filename = filename_prefix..subfile
+ local cbfunc = ser.write_to_fd
+ if callbacks_table and callbacks_table[subfile] then
+ cbfunc = callbacks_table[subfile]
+ end
+
+ local success = false
+ local file, ret = io.open(filename..".new", "w")
+ if file then
+ -- save the file using the callback
+ success, ret = pcall(cbfunc, data, file, config)
+ end
+
+ if not success then
+ serialize_lib.log_error("Unable to save data to '"..filename..".new':")
+ serialize_lib.log_error(ret)
+ return nil, ret
+ end
+ end
+
+ local first_error
+ for file, _ in pairs(parts_table) do
+ local filename = filename_prefix..file
+ local succ, err = save_atomic_move_file(filename)
+ if not succ and not first_error then
+ first_error = err
+ end
+ end
+
+ return not first_error, first_error -- either true,nil or nil,error
+end
+
+
diff --git a/serialize_lib/init.lua b/serialize_lib/init.lua
index 7a1a10b..20ffa4d 100644
--- a/serialize_lib/init.lua
+++ b/serialize_lib/init.lua
@@ -30,63 +30,45 @@ Not all functions use all of the parameters, so you can simplify your config som
function serialize_lib.log_error(text)
- minetest.log("error", "[serialize_lib] "..text)
+ minetest.log("error", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
end
function serialize_lib.log_warn(text)
- minetest.log("warning", "[serialize_lib] "..text)
+ minetest.log("warning", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
end
function serialize_lib.log_info(text)
- minetest.log("action", "[serialize_lib] "..text)
+ minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
end
function serialize_lib.log_debug(text)
- minetest.log("action", "[serialize_lib](debug) "..text)
+ minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?")..") DEBUG: "..(text or "<nil>"))
end
-- basic serialization/deserialization
-- ===================================
-local ser = dofile("serialize.lua")
+local mp = minetest.get_modpath(minetest.get_current_modname())
+serialize_lib.serialize = dofile(mp.."/serialize.lua")
+dofile(mp.."/atomic.lua")
+
+local ser = serialize_lib.serialize
-- Opens the passed filename, and returns deserialized table
-- When an error occurs, logs an error and returns false
function serialize_lib.read_table_from_file(filename)
- local succ, err = pcall(ser.read_from_file, filename)
+ local succ, ret = pcall(ser.read_from_file, filename)
if not succ then
- serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err)
+ serialize_lib.log_error(ret)
end
- return succ
+ return ret
end
-- Writes table into file
-- When an error occurs, logs an error and returns false
-function serialize_lib.write_table_to_file(filename)
- local succ, err = pcall(ser.write_to_file, filename)
+function serialize_lib.write_table_to_file(root_table, filename)
+ local succ, ret = pcall(ser.write_to_file, root_table, filename)
if not succ then
- serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err)
+ serialize_lib.log_error(ret)
end
- return succ
+ return ret
end
--- Managing files and backups
--- ==========================
-
---[[
-The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within
-the write, so we have incomplete data. So, the following scheme is applied:
-1. writes to <filename>.new (if .new already exists, try to complete the moving first)
-2. moves <filename> to <filename>.old, possibly overwriting an existing file (special windows behavior)
-3. moves <filename>.new to <filename>
-
-During loading, we apply the following order of precedence:
-1. <filename>.new
-2. <filename>
-3. <filename>.old
-
-Normal case: <filename> and <filename>.old exist, loading <filename>
-Interrupted during write: .new is damaged, loads last regular state
-Interrupted during the move operations: either <filename>.new or <filename> represents the latest state
-Other corruption: at least the .old state may still be present
-
-]]--
-
diff --git a/serialize_lib/serialize.lua b/serialize_lib/serialize.lua
index 8ffd917..2d7c3a0 100644
--- a/serialize_lib/serialize.lua
+++ b/serialize_lib/serialize.lua
@@ -55,7 +55,7 @@ function write_table(t, file, config)
if istable then
vs = "T"
- if config.skip_empty_tables then
+ if config and config.skip_empty_tables then
writeit = not table_is_empty(value)
end
else
@@ -75,6 +75,7 @@ end
function value_to_string(t)
if type(t)=="table" then
+ file:close()
error("Can not serialize a table in the key position!")
elseif type(t)=="boolean" then
if t then
@@ -87,6 +88,7 @@ function value_to_string(t)
elseif type(t)=="string" then
return "S"..escape_chars(t)
else
+ file:close()
error("Can not serialize '"..type(t).."' type!")
end
return str
@@ -108,6 +110,7 @@ function read_table(t, file)
while true do
line = file:read("*l")
if not line then
+ file:close()
error("Unexpected EOF or read error!")
end
@@ -117,6 +120,7 @@ function read_table(t, file)
end
ks, vs = string.match(line, "^(.+[^&]):(.+)$")
if not ks or not vs then
+ file:close()
error("Unable to parse line: '"..line.."'!")
end
kv = string_to_value(ks)
@@ -137,6 +141,7 @@ function string_to_value(str, table_allow)
if table_allow then
return {}, true
else
+ file:close()
error("Table not allowed in key component!")
end
elseif first=="N" then
@@ -144,6 +149,7 @@ function string_to_value(str, table_allow)
if num then
return num
else
+ file:close()
error("Unable to parse number: '"..rest.."'!")
end
elseif first=="B" then
@@ -152,11 +158,13 @@ function string_to_value(str, table_allow)
elseif rest=="1" then
return true
else
+ file:close()
error("Unable to parse boolean: '"..rest.."'!")
end
elseif first=="S" then
return unescape_chars(rest)
else
+ file:close()
error("Unknown literal type '"..first.."' for literal '"..str.."'!")
end
end
@@ -177,20 +185,20 @@ config = {
}
]]
--- Writes the passed table into the passed file descriptor, and closes the file afterwards
+-- Writes the passed table into the passed file descriptor, and closes the file
local function write_to_fd(root_table, file, config)
file:write("LUA_SER v=1\n")
write_table(root_table, file, config)
file:write("E\nEND_SER\n")
- file:close()
end
-- Reads the file contents from the passed file descriptor and returns the table on success
--- Throws errors when something is wrong.
+-- Throws errors when something is wrong. Closes the file.
-- config: see above
local function read_from_fd(file)
local first_line = file:read("*l")
if first_line ~= "LUA_SER v=1" then
+ file:close()
error("Expected header, got '"..first_line.."' instead!")
end
local t = {}
@@ -209,7 +217,7 @@ function write_to_file(root_table, filename, config)
-- try opening the file
local file, err = io.open(filename, "w")
if not file then
- error("Failed opening file '"..filename.."' for write: "..err)
+ error("Failed opening file '"..filename.."' for write:\n"..err)
end
write_to_fd(root_table, file, config)
@@ -221,7 +229,7 @@ function read_from_file(filename)
-- try opening the file
local file, err = io.open(filename, "r")
if not file then
- error("Failed opening file '"..filename.."' for read: "..err)
+ error("Failed opening file '"..filename.."' for read:\n"..err)
end
return read_from_fd(file)
diff --git a/serialize_lib/settingtypes.txt b/serialize_lib/settingtypes.txt
new file mode 100644
index 0000000..3a565a6
--- /dev/null
+++ b/serialize_lib/settingtypes.txt
@@ -0,0 +1,12 @@
+# Enable strict file loading mode
+# If enabled, if any error occurs during loading of a file using the 'atomic' API,
+# an error is thrown. You probably need to disable this option for initial loading after
+# creating the world.
+serialize_lib_strict_loading (Strict loading) bool false
+
+# Do not automatically switch to "Windows mode" when saving atomically
+# Normally, when renaming <filename>.new to <filename> fails, serialize_lib automatically
+# switches to a mode where it deletes <filename> prior to moving. Enable this option to prevent
+# this behavior and abort saving instead.
+serialize_lib_no_auto_windows_mode (No automatic Windows Mode) bool false
+