Serialize_lib: finish up and add atomic api

4 files changed, 257 insertions, 40 deletions

diff --git a/serialize_lib/atomic.lua b/serialize_lib/atomic.lua
new file mode 100644
index 0000000..85937db
--- /dev/null
+++ b/serialize_lib/atomic.lua
@@ -0,0 +1,215 @@
+-- atomic.lua
+-- Utilities for transaction-like handling of serialized state files
+-- Also for multiple files that must be synchronous, as advtrains currently requires.
+
+
+-- Managing files and backups
+-- ==========================
+
+--[[
+The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within
+the write, so we have incomplete data. So, the following scheme is applied:
+Unix:
+1. writes to <filename>.new
+2. moves <filename>.new to <filename>, clobbering previous file
+Windows:
+1. writes to <filename>.new
+2. delete <filename>
+3. moves <filename>.new to <filename>
+
+We count a new version of the state as "committed" after stage 2.
+
+During loading, we apply the following order of precedence:
+1. <filename>
+2. <filename>.new (windows only, in case we were interrupted just before 3. when saving)
+
+
+All of these functions return either true on success or nil, error on error.
+]]--
+
+local ser = serialize_lib.serialize
+
+local windows_mode = false
+
+-- == local functions ==
+
+local function save_atomic_move_file(filename)
+	--2. if windows mode, delete main file
+	if windows_mode then
+		local delsucc, err = os.remove(filename)
+		if not delsucc then
+			serialize_lib.log_error("Unable to delete old savefile '"..filename.."':")
+			serialize_lib.log_error(err)
+			return nil, err
+		end
+	end
+	
+	--3. move file
+	local mvsucc, err = os.rename(filename..".new", filename)
+	if not mvsucc then
+		if minetest.settings:get_bool("serialize_lib_no_auto_windows_mode") or windows_mode then
+			serialize_lib.log_error("Unable to move '"..filename.."':")
+			serialize_lib.log_error(err)
+			return nil, err
+		else
+			-- enable windows mode and try again
+			serialize_lib.log_info("Enabling Windows mode for atomic saving...")
+			windows_mode = true
+			return save_atomic_move_file(filename)
+		end
+	end
+	
+	return true
+end
+
+local function open_file_and_save_callback(callback, filename)
+	local file, err = io.open(filename, "w")
+	if not file then
+		error("Failed opening file '"..filename.."' for write:\n"..err)
+	end
+	
+	callback(file)
+	return true
+end
+
+local function open_file_and_load_callback(filename, callback)
+	local file, err = io.open(filename, "r")
+	if not file then
+		error("Failed opening file '"..filename.."' for read:\n"..err)
+	end
+	
+	return callback(file)
+end
+
+-- == public functions ==
+
+-- Load a saved state (according to comment above)
+-- if 'callback' is nil: reads serialized table.
+-- returns the read table, or nil,err on error
+-- if 'callback' is a function (signature func(file_handle) ):
+-- Counterpart to save_atomic with function argument. Opens the file and calls callback on it.
+-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
+-- The callback's first return value is returned by load_atomic
+function serialize_lib.load_atomic(filename, callback)
+	
+	local cbfunc = callback or ser.read_from_fd
+	
+	-- try <filename>
+	local file, ret = io.open(filename, "r")
+	if file then
+		-- read the file using the callback
+		local success
+		success, ret = pcall(cbfunc, file)
+		if success then
+			return ret
+		end
+	end
+	
+	if minetest.settings:get_bool("serialize_lib_strict_loading") then
+		serialize_lib.save_lock = true
+		error("Loading data from file '"..filename.."' failed:\n"
+				..ret.."\nDisable Strict Loading to ignore.")
+	end
+		
+	serialize_lib.log_warn("Loading data from file '"..filename.."' failed, trying .new fallback:")
+	serialize_lib.log_warn(ret)
+	
+	-- try <filename>.new
+	file, ret = io.open(filename..".new", "r")
+	if file then
+		-- read the file using the callback
+		local success
+		success, ret = pcall(cbfunc, file)
+		if success then
+			return ret
+		end
+	end
+	
+	serialize_lib.log_error("Unable to load data from '"..filename..".new':")
+	serialize_lib.log_error(ret)
+	serialize_lib.log_error("Note: This message is normal when the mod is loaded the first time on this world.")
+	
+	return nil, ret
+end
+
+-- Save a file atomically (as described above)
+-- 'data' is the data to be saved (when a callback is used, this can be nil)
+-- if 'callback' is nil:
+-- data must be a table, and is serialized into the file
+-- if 'callback' is a function (signature func(data, file_handle) ):
+-- Opens the file and calls callback on it. The 'data' argument is the data passed to save_atomic().
+-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
+-- The callback's first return value is returned by load_atomic
+-- Important: the callback must close the file in all cases!
+function serialize_lib.save_atomic(data, filename, callback, config)
+	if serialize_lib.save_lock then
+		serialize_lib.log_warn("Instructed to save '"..filename.."', but save lock is active!")
+		return nil
+	end
+	
+	local cbfunc = callback or ser.write_to_fd
+
+	local file, ret = io.open(filename..".new", "w")
+	if file then
+		-- save the file using the callback
+		local success
+		success, ret = pcall(cbfunc, data, file)
+		if success then
+			return save_atomic_move_file(filename)
+		end
+	end
+	serialize_lib.log_error("Unable to save data to '"..filename..".new':")
+	serialize_lib.log_error(ret)
+	return nil, ret
+end
+
+
+-- Saves multiple files synchronously. First writes all data to all <filename>.new files,
+-- then moves all files in quick succession to avoid inconsistent backups.
+-- parts_table is a table where the keys are used as part of the filename and the values
+-- are the respective data written to it.
+-- e.g. if parts_table={foo={...}, bar={...}}, then <filename_prefix>foo and <filename_prefix>bar are written out.
+-- if 'callbacks_table' is defined, it is consulted for callbacks the same way save_atomic does.
+-- example: if callbacks_table = {foo = func()...}, then the callback is used during writing of file 'foo' (but not for 'bar')
+-- Note however that you must at least insert a "true" in the parts_table if you don't use the data argument.
+-- Important: the callback must close the file in all cases!
+function serialize_lib.save_atomic_multiple(parts_table, filename_prefix, callbacks_table, config)
+	if serialize_lib.save_lock then
+		serialize_lib.log_warn("Instructed to save '"..filename_prefix.."' (multiple), but save lock is active!")
+		return nil
+	end
+	
+	for subfile, data in pairs(parts_table) do
+		local filename = filename_prefix..subfile
+		local cbfunc = ser.write_to_fd
+		if callbacks_table and callbacks_table[subfile] then
+			cbfunc = callbacks_table[subfile]
+		end
+		
+		local success = false
+		local file, ret = io.open(filename..".new", "w")
+		if file then
+			-- save the file using the callback
+			success, ret = pcall(cbfunc, data, file, config)
+		end
+		
+		if not success then
+			serialize_lib.log_error("Unable to save data to '"..filename..".new':")
+			serialize_lib.log_error(ret)
+			return nil, ret
+		end
+	end
+	
+	local first_error
+	for file, _ in pairs(parts_table) do
+		local filename = filename_prefix..file
+		local succ, err = save_atomic_move_file(filename)
+		if not succ and not first_error then
+			first_error = err
+		end
+	end
+	
+	return not first_error, first_error -- either true,nil or nil,error
+end
+
+
diff --git a/serialize_lib/init.lua b/serialize_lib/init.lua
index 7a1a10b..20ffa4d 100644
--- a/serialize_lib/init.lua
+++ b/serialize_lib/init.lua
@@ -30,63 +30,45 @@ Not all functions use all of the parameters, so you can simplify your config som
 
 
 function serialize_lib.log_error(text)
-	minetest.log("error", "[serialize_lib] "..text)
+	minetest.log("error", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
 end
 function serialize_lib.log_warn(text)
-	minetest.log("warning", "[serialize_lib] "..text)
+	minetest.log("warning", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
 end
 function serialize_lib.log_info(text)
-	minetest.log("action", "[serialize_lib] "..text)
+	minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
 end
 function serialize_lib.log_debug(text)
-	minetest.log("action", "[serialize_lib](debug) "..text)
+	minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?")..") DEBUG: "..(text or "<nil>"))
 end
 
 -- basic serialization/deserialization
 -- ===================================
 
-local ser = dofile("serialize.lua")
+local mp = minetest.get_modpath(minetest.get_current_modname())
+serialize_lib.serialize = dofile(mp.."/serialize.lua")
+dofile(mp.."/atomic.lua")
+
+local ser = serialize_lib.serialize
 
 -- Opens the passed filename, and returns deserialized table
 -- When an error occurs, logs an error and returns false
 function serialize_lib.read_table_from_file(filename)
-	local succ, err = pcall(ser.read_from_file, filename)
+	local succ, ret = pcall(ser.read_from_file, filename)
 	if not succ then
-		serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err)
+		serialize_lib.log_error(ret)
 	end
-	return succ
+	return ret
 end
 
 -- Writes table into file
 -- When an error occurs, logs an error and returns false
-function serialize_lib.write_table_to_file(filename)
-	local succ, err = pcall(ser.write_to_file, filename)
+function serialize_lib.write_table_to_file(root_table, filename)
+	local succ, ret = pcall(ser.write_to_file, root_table, filename)
 	if not succ then
-		serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err)
+		serialize_lib.log_error(ret)
 	end
-	return succ
+	return ret
 end
 
--- Managing files and backups
--- ==========================
-
---[[
-The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within
-the write, so we have incomplete data. So, the following scheme is applied:
-1. writes to <filename>.new (if .new already exists, try to complete the moving first)
-2. moves <filename> to <filename>.old, possibly overwriting an existing file (special windows behavior)
-3. moves <filename>.new to <filename>
-
-During loading, we apply the following order of precedence:
-1. <filename>.new
-2. <filename>
-3. <filename>.old
-
-Normal case: <filename> and <filename>.old exist, loading <filename>
-Interrupted during write: .new is damaged, loads last regular state
-Interrupted during the move operations: either <filename>.new or <filename> represents the latest state
-Other corruption: at least the .old state may still be present
-
-]]--
-
 
diff --git a/serialize_lib/serialize.lua b/serialize_lib/serialize.lua
index 8ffd917..2d7c3a0 100644
--- a/serialize_lib/serialize.lua
+++ b/serialize_lib/serialize.lua
@@ -55,7 +55,7 @@ function write_table(t, file, config)
 		
 		if istable then
 			vs = "T"
-			if config.skip_empty_tables then
+			if config and config.skip_empty_tables then
 				writeit = not table_is_empty(value)
 			end
 		else
@@ -75,6 +75,7 @@ end
 
 function value_to_string(t)
 	if type(t)=="table" then
+		file:close()
 		error("Can not serialize a table in the key position!")
 	elseif type(t)=="boolean" then
 		if t then
@@ -87,6 +88,7 @@ function value_to_string(t)
 	elseif type(t)=="string" then
 		return "S"..escape_chars(t)
 	else
+		file:close()
 		error("Can not serialize '"..type(t).."' type!")
 	end
 	return str
@@ -108,6 +110,7 @@ function read_table(t, file)
 	while true do
 		line = file:read("*l")
 		if not line then
+			file:close()
 			error("Unexpected EOF or read error!")
 		end
 		
@@ -117,6 +120,7 @@ function read_table(t, file)
 		end
 		ks, vs = string.match(line, "^(.+[^&]):(.+)$")
 		if not ks or not vs then
+			file:close()
 			error("Unable to parse line: '"..line.."'!")
 		end
 		kv = string_to_value(ks)
@@ -137,6 +141,7 @@ function string_to_value(str, table_allow)
 		if table_allow then
 			return {}, true
 		else
+			file:close()
 			error("Table not allowed in key component!")
 		end
 	elseif first=="N" then
@@ -144,6 +149,7 @@ function string_to_value(str, table_allow)
 		if num then
 			return num
 		else
+			file:close()
 			error("Unable to parse number: '"..rest.."'!")
 		end
 	elseif first=="B" then
@@ -152,11 +158,13 @@ function string_to_value(str, table_allow)
 		elseif rest=="1" then
 			return true
 		else
+			file:close()
 			error("Unable to parse boolean: '"..rest.."'!")
 		end
 	elseif first=="S" then
 		return unescape_chars(rest)
 	else
+		file:close()
 		error("Unknown literal type '"..first.."' for literal '"..str.."'!")
 	end
 end
@@ -177,20 +185,20 @@ config = {
 }
 ]]
 
--- Writes the passed table into the passed file descriptor, and closes the file afterwards
+-- Writes the passed table into the passed file descriptor, and closes the file
 local function write_to_fd(root_table, file, config)
 	file:write("LUA_SER v=1\n")
 	write_table(root_table, file, config)
 	file:write("E\nEND_SER\n")
-	file:close()
 end
 
 -- Reads the file contents from the passed file descriptor and returns the table on success
--- Throws errors when something is wrong.
+-- Throws errors when something is wrong. Closes the file.
 -- config: see above
 local function read_from_fd(file)
 	local first_line = file:read("*l")
 	if first_line ~= "LUA_SER v=1" then
+		file:close()
 		error("Expected header, got '"..first_line.."' instead!")
 	end
 	local t = {}
@@ -209,7 +217,7 @@ function write_to_file(root_table, filename, config)
 	-- try opening the file
 	local file, err = io.open(filename, "w")
 	if not file then
-		error("Failed opening file '"..filename.."' for write: "..err)
+		error("Failed opening file '"..filename.."' for write:\n"..err)
 	end
 	
 	write_to_fd(root_table, file, config)
@@ -221,7 +229,7 @@ function read_from_file(filename)
 	-- try opening the file
 	local file, err = io.open(filename, "r")
 	if not file then
-		error("Failed opening file '"..filename.."' for read: "..err)
+		error("Failed opening file '"..filename.."' for read:\n"..err)
 	end
 	
 	return read_from_fd(file)
diff --git a/serialize_lib/settingtypes.txt b/serialize_lib/settingtypes.txt
new file mode 100644
index 0000000..3a565a6
--- /dev/null
+++ b/serialize_lib/settingtypes.txt
@@ -0,0 +1,12 @@
+# Enable strict file loading mode
+# If enabled, if any error occurs during loading of a file using the 'atomic' API,
+# an error is thrown. You probably need to disable this option for initial loading after
+# creating the world.
+serialize_lib_strict_loading (Strict loading) bool false
+
+# Do not automatically switch to "Windows mode" when saving atomically
+# Normally, when renaming <filename>.new to <filename> fails, serialize_lib automatically
+# switches to a mode where it deletes <filename> prior to moving. Enable this option to prevent
+# this behavior and abort saving instead.
+serialize_lib_no_auto_windows_mode (No automatic Windows Mode) bool false
+