Updated to be compatible with new advtrains file structureHEADmaster

May be hacky, but works.

4 files changed, 338 insertions, 208 deletions

diff --git a/main.lua b/main.lua
index 8ff86cd..93dadd9 100644
--- a/main.lua
+++ b/main.lua
@@ -85,7 +85,7 @@ function atdump(t, intend)
 end
 
 dofile("vector.lua")
-dofile("serialize.lua")
+local serialize = dofile("serialize.lua")
 dofile("helpers.lua")
 dofile("tracks.lua")
 dofile("track_defs.lua")
@@ -128,22 +128,25 @@ end
 datapath, mappath, no_trains, worldimage = parse_args(arg)
 
 -- Load saves
-local file, err = io.open(datapath.."advtrains_trains", "r")
-local tbl = minetest.deserialize(file:read("*a"))
+local tbl = serialize.read_from_file(datapath.."advtrains_core.ls")
+--local file, err = io.open(datapath.."advtrains_trains", "r")
+--local tbl = minetest.deserialize(file:read("*a"))
 if type(tbl) ~= "table" then
 	error("Trains file: not a table")
 end
-advtrains.trains = tbl
-file:close()
+advtrains.trains = tbl.trains
+--file:close()
 
 --ndb contains the defs, while ndb2 is the actual contents
-file, err = io.open(datapath.."advtrains_ndb", "r")
-tbl = minetest.deserialize(file:read("*a"))
-if type(tbl) ~= "table" then
-	error("Node database file: not a table")
-end
-advtrains.ndb.load_data(tbl)
-file:close()
+dofile("nodedb.lua")
+local file, err = io.open(datapath.."advtrains_ndb4.ls", "r")
+--tbl = minetest.deserialize(file:read("*a"))
+--if type(tbl) ~= "table" then
+	--error("Node database file: not a table")
+--end
+--advtrains.ndb.load_data(tbl)
+advtrains.ndb.load_callback(file)
+--file:close()
 
 -- open svg file
 
diff --git a/nodedb.lua b/nodedb.lua
index de582f0..6f852d6 100644
--- a/nodedb.lua
+++ b/nodedb.lua
@@ -31,6 +31,7 @@ local ndb={}
 --local variables for performance
 local ndb_nodeids={}
 local ndb_nodes={}
+local ndb_ver
 
 local function ndbget(x,y,z)
 	local ny=ndb_nodes[y]
@@ -52,22 +53,39 @@ local function ndbset(x,y,z,v)
 	ndb_nodes[y][x][z]=v
 end
 
+-- load/save
 
-local path="advtrains_ndb2"
---load
+local path_pre_v4=datapath.."advtrains_ndb2"
+--load pre_v4 format
 --nodeids get loaded by advtrains init.lua and passed here
-function ndb.load_data(data)
+function ndb.load_data_pre_v4(data)
+	print("nodedb: Loading pre v4 format")
+
 	ndb_nodeids = data and data.nodeids or {}
-	local file, err = io.open(datapath..path, "rb")
+	ndb_ver = data and data.ver or 0
+	if ndb_ver < 1 then
+		for k,v in pairs(ndb_nodeids) do
+			if v == "advtrains:dtrack_xing4590_st" then
+				cidDepr = k
+			elseif v == "advtrains:dtrack_xing90plusx_45l" then
+				cidNew = k
+			end
+		end
+	end
+	local file, err = io.open(path_pre_v4, "rb")
 	if not file then
 		print("Couldn't load the node database: ", err or "Unknown Error")
 	else
+		-- Note: code duplication because of weird coordinate order in ndb2 format (z,y,x)
 		local cnt=0
 		local hst_z=file:read(2)
 		local hst_y=file:read(2)
 		local hst_x=file:read(2)
 		local cid=file:read(2)
 		while hst_z and hst_y and hst_x and cid and #hst_z==2 and #hst_y==2 and #hst_x==2 and #cid==2 do
+			if (ndb_ver < 1 and cid == cidDepr) then
+				cid = cidNew
+			end
 			ndbset(bytes_to_int(hst_x), bytes_to_int(hst_y), bytes_to_int(hst_z), bytes_to_int(cid))
 			cnt=cnt+1
 			hst_z=file:read(2)
@@ -75,12 +93,66 @@ function ndb.load_data(data)
 			hst_x=file:read(2)
 			cid=file:read(2)
 		end
-		print("nodedb: read", cnt, "nodes.")
+		print("nodedb (ndb2 format): read", cnt, "nodes.")
 		ndb_nodes_total = cnt
 		file:close()
 	end
+	ndb_ver = 1
+end
+
+-- the new ndb file format is backported from cellworld, and stores the cids also in the ndb file.
+-- These functions have the form of a serialize_lib atomic load/save callback and are called from avt_save/avt_load.
+function ndb.load_callback(file)
+	-- read version
+	local vers_byte = file:read(1)
+	local version = string.byte(vers_byte)
+	if version~=1 then
+		file:close()
+		error("Doesn't support v4 nodedb file of version "..version)
+	end
+	
+	-- read cid mappings
+	local nstr_byte = file:read(2)
+	local nstr = bytes_to_int(nstr_byte)
+	for i = 1,nstr do
+		local stid_byte = file:read(2)
+		local stid = bytes_to_int(stid_byte)
+		local stna = file:read("*l")
+		--atdebug("content id:", stid, "->", stna)
+		ndb_nodeids[stid] = stna
+	end
+	print("[nodedb] read", nstr, "node content ids.")
+
+	-- read nodes
+	local cnt=0
+	local hst_x=file:read(2)
+	local hst_y=file:read(2)
+	local hst_z=file:read(2)
+	local cid=file:read(2)
+	local cidi
+	while hst_z and hst_y and hst_x and cid and #hst_z==2 and #hst_y==2 and #hst_x==2 and #cid==2 do
+		cidi = bytes_to_int(cid)
+		-- prevent file corruption already here
+		if not ndb_nodeids[u14b(cidi)] then
+			-- clear the ndb data, to reinitialize it
+			-- in strict loading mode, doesn't matter as starting will be interrupted anyway
+			ndb_nodeids = {}
+			ndb_nodes = {}
+			error("NDB file is corrupted (found entry with invalid cid)")
+		end
+		ndbset(bytes_to_int(hst_x), bytes_to_int(hst_y), bytes_to_int(hst_z), cidi)
+		cnt=cnt+1
+		hst_x=file:read(2)
+		hst_y=file:read(2)
+		hst_z=file:read(2)
+		cid=file:read(2)
+	end
+	print("[nodedb] read", cnt, "nodes.")
+	ndb_nodes_total = cnt
+	file:close()
 end
 
+
 --function to get node. track database is not helpful here.
 function ndb.get_node_or_nil(pos)
 	-- FIX for bug found on linuxworks server:
@@ -124,6 +196,7 @@ end
 --false  if it's not a rail or the train does not drive on this rail, but it is loaded or
 --nil    if the node is neither loaded nor in trackdb
 --the distraction between false and nil will be needed only in special cases.(train initpos)
+advtrains = advtrains or {}
 function advtrains.get_rail_info_at(pos)
 	local rdp=advtrains.round_vector_floor_y(pos)
 	
diff --git a/routes.lua b/routes.lua
index 04b96fe..e828b67 100755
--- a/routes.lua
+++ b/routes.lua
@@ -8,7 +8,7 @@ math.hypot = function(a,b) return math.sqrt(a*a + b*b) end
 function attrans(str) return str end
 
 dofile("vector.lua")
-dofile("serialize.lua")
+local serialize = dofile("serialize.lua")
 dofile("helpers.lua")
 
 
@@ -72,10 +72,11 @@ function ars_to_text(arstab)
 end
 
 
-local file = io.open(datapath.."advtrains_interlocking_tcbs", "r")
-local tbl = minetest.deserialize(file:read("*a"))
-advtrains.tcbs = tbl
-file:close()
+--local file = io.open(datapath.."advtrains_interlocking_tcbs", "r")
+--local tbl = minetest.deserialize(file:read("*a"))
+local tbl = serialize.read_from_file(datapath.."advtrains_interlocking")
+advtrains.tcbs = tbl.tcbs
+--file:close()
 
 local jsonfile = io.open(datapath.."signals.json", "w")
 
diff --git a/serialize.lua b/serialize.lua
index 692ddd5..4b1ebb7 100755
--- a/serialize.lua
+++ b/serialize.lua
@@ -1,221 +1,274 @@
---- Lua module to serialize values as Lua code.
--- From: https://github.com/fab13n/metalua/blob/no-dll/src/lib/serialize.lua
--- License: MIT
--- @copyright 2006-2997 Fabien Fleutot <metalua@gmail.com>
--- @author Fabien Fleutot <metalua@gmail.com>
--- @author ShadowNinja <shadowninja@minetest.net>
---------------------------------------------------------------------------------
-
---- Serialize an object into a source code string. This string, when passed as
--- an argument to deserialize(), returns an object structurally identical to
--- the original one.  The following are currently supported:
---   * Booleans, numbers, strings, and nil.
---   * Functions; uses interpreter-dependent (and sometimes platform-dependent) bytecode!
---   * Tables; they can cantain multiple references and can be recursive, but metatables aren't saved.
--- This works in two phases:
---   1. Recursively find and record multiple references and recursion.
---   2. Recursively dump the value into a string.
--- @param x Value to serialize (nil is allowed).
--- @return load()able string containing the value.
-function core.serialize(x)
-	local local_index  = 1  -- Top index of the "_" local table in the dump
-	-- table->nil/1/2 set of tables seen.
-	-- nil = not seen, 1 = seen once, 2 = seen multiple times.
-	local seen = {}
-
-	-- nest_points are places where a table appears within itself, directly
-	-- or not.  For instance, all of these chunks create nest points in
-	-- table x: "x = {}; x[x] = 1", "x = {}; x[1] = x",
-	-- "x = {}; x[1] = {y = {x}}".
-	-- To handle those, two tables are used by mark_nest_point:
-	-- * nested - Transient set of tables being currently traversed.
-	--   Used for detecting nested tables.
-	-- * nest_points - parent->{key=value, ...} table cantaining the nested
-	--   keys and values in the parent.  They're all dumped after all the
-	--   other table operations have been performed.
-	--
-	-- mark_nest_point(p, k, v) fills nest_points with information required
-	-- to remember that key/value (k, v) creates a nest point  in table
-	-- parent. It also marks "parent" and the nested item(s) as occuring
-	-- multiple times, since several references to it will be required in
-	-- order to patch the nest points.
-	local nest_points  = {}
-	local nested = {}
-	local function mark_nest_point(parent, k, v)
-		local nk, nv = nested[k], nested[v]
-		local np = nest_points[parent]
-		if not np then
-			np = {}
-			nest_points[parent] = np
-		end
-		np[k] = v
-		seen[parent] = 2
-		if nk then seen[k] = 2 end
-		if nv then seen[v] = 2 end
+-- serialize.lua
+-- Lua-conformant library file that has no minetest dependencies
+-- Contains the serialization and deserialization routines
+
+--[[
+Version history:
+1 - initial
+2 - also escaping CR character as &r
+
+Structure of entry:
+[keytype][key]:[valuetype][val]
+Types:
+	B - bool
+		-> 0=false, 1=true
+    S - string
+		-> see below
+    N - number
+		-> thing compatible with tonumber()
+Table:
+[keytype][key]:T
+... content is nested in table until the matching
+E
+
+example:
+LUA_SER v=2		{
+Skey:Svalue			key = "value",
+N1:Seins			[1] = "eins",
+B1:T				[true] = {
+Sa:Sb					a = "b",
+Sc:B0					c = false,
+E					}
+E				}
+
+String representations:
+In strings the following characters are escaped by &
+'&' -> '&&'
+(line break) -> '&n'
+(CR) -> '&r'
+':' -> '&:'
+All other characters are unchanged as they bear no special meaning.
+]]
+
+local write_table, literal_to_string, escape_chars, table_is_empty
+
+function table_is_empty(t)
+	for _,_ in pairs(t) do
+		return false
 	end
+	return true
+end
 
-	-- First phase, list the tables and functions which appear more than
-	-- once in x.
-	local function mark_multiple_occurences(x)
-		local tp = type(x)
-		if tp ~= "table" and tp ~= "function" then
-			-- No identity (comparison is done by value, not by instance)
-			return
+function write_table(t, file, config)
+	local ks, vs, writeit, istable
+	for key, value in pairs(t) do
+		ks = value_to_string(key, false)
+		writeit = true
+		istable = type(value)=="table"
+		
+		if istable then
+			vs = "T"
+			if config and config.skip_empty_tables then
+				writeit = not table_is_empty(value)
+			end
+		else
+			vs = value_to_string(value, true)
 		end
-		if seen[x] == 1 then
-			seen[x] = 2
-		elseif seen[x] ~= 2 then
-			seen[x] = 1
+		
+		if writeit then
+			file:write(ks..":"..vs.."\n")
+			
+			if istable then
+				write_table(value, file, config)
+				file:write("E\n")
+			end
 		end
+	end
+end
 
-		if tp == "table" then
-			nested[x] = true
-			for k, v in pairs(x) do
-				if nested[k] or nested[v] then
-					mark_nest_point(x, k, v)
-				else
-					mark_multiple_occurences(k)
-					mark_multiple_occurences(v)
-				end
-			end
-			nested[x] = nil
+function value_to_string(t)
+	if type(t)=="table" then
+		file:close()
+		error("Can not serialize a table in the key position!")
+	elseif type(t)=="boolean" then
+		if t then
+			return "B1"
+		else
+			return "B0"
 		end
+	elseif type(t)=="number" then
+		return "N"..t
+	elseif type(t)=="string" then
+		return "S"..escape_chars(t)
+	else
+		--error("Can not serialize '"..type(t).."' type!")
+		return "S<function>"
 	end
+	return str
+end
 
-	local dumped     = {}  -- object->varname set
-	local local_defs = {}  -- Dumped local definitions as source code lines
+function escape_chars(str)
+	local rstr = string.gsub(str, "&", "&&")
+	rstr = string.gsub(rstr, ":", "&:")
+	rstr = string.gsub(rstr, "\r", "&r")
+	rstr = string.gsub(rstr, "\n", "&n")
+	return rstr
+end
+
+------
 
-	-- Mutually recursive local functions:
-	local dump_val, dump_or_ref_val
+local read_table, string_to_value, unescape_chars
 
-	-- If x occurs multiple times, dump the local variable rather than
-	-- the value. If it's the first time it's dumped, also dump the
-	-- content in local_defs.
-	function dump_or_ref_val(x)
-		if seen[x] ~= 2 then
-			return dump_val(x)
+function read_table(t, file)
+	local line, ks, vs, kv, vv, vt
+	while true do
+		line = file:read("*l")
+		if not line then
+			file:close()
+			error("Unexpected EOF or read error!")
+		end
+		
+		if line=="E" then
+			-- done with this table
+			return
+		end
+		ks, vs = string.match(line, "^(.*[^&]):(.+)$")
+		if not ks or not vs then
+			file:close()
+			error("Unable to parse line: '"..line.."'!")
 		end
-		local var = dumped[x]
-		if var then  -- Already referenced
-			return var
+		kv = string_to_value(ks)
+		vv, vt = string_to_value(vs, true)
+		if vt then
+			read_table(vv, file)
 		end
-		-- First occurence, create and register reference
-		local val = dump_val(x)
-		local i = local_index
-		local_index = local_index + 1
-		var = "_["..i.."]"
-		local_defs[#local_defs + 1] = var.." = "..val
-		dumped[x] = var
-		return var
+		-- put read value in table
+		t[kv] = vv
 	end
+end
 
-	-- Second phase.  Dump the object; subparts occuring multiple times
-	-- are dumped in local variables which can be referenced multiple
-	-- times.  Care is taken to dump local vars in a sensible order.
-	function dump_val(x)
-		local  tp = type(x)
-		if     x  == nil        then return "nil"
-		elseif tp == "string"   then return string.format("%q", x)
-		elseif tp == "boolean"  then return x and "true" or "false"
-		elseif tp == "function" then
-			return string.format("loadstring(%q)", string.dump(x))
-		elseif tp == "number"   then
-			-- Serialize integers with string.format to prevent
-			-- scientific notation, which doesn't preserve
-			-- precision and breaks things like node position
-			-- hashes.  Serialize floats normally.
-			if math.floor(x) == x then
-				return string.format("%d", x)
-			else
-				return tostring(x)
-			end
-		elseif tp == "table" then
-			local vals = {}
-			local idx_dumped = {}
-			local np = nest_points[x]
-			for i, v in ipairs(x) do
-				if not np or not np[i] then
-					vals[#vals + 1] = dump_or_ref_val(v)
-				end
-				idx_dumped[i] = true
-			end
-			for k, v in pairs(x) do
-				if (not np or not np[k]) and
-						not idx_dumped[k] then
-					vals[#vals + 1] = "["..dump_or_ref_val(k).."] = "
-						..dump_or_ref_val(v)
-				end
-			end
-			return "{"..table.concat(vals, ", ").."}"
+-- returns: value, is_table
+function string_to_value(str, table_allow)
+	local first = string.sub(str, 1,1)
+	local rest = string.sub(str, 2)
+	if first=="T" then
+		if table_allow then
+			return {}, true
 		else
-			error("Can't serialize data of type "..tp)
+			file:close()
+			error("Table not allowed in key component!")
 		end
-	end
-
-	local function dump_nest_points()
-		for parent, vals in pairs(nest_points) do
-			for k, v in pairs(vals) do
-				local_defs[#local_defs + 1] = dump_or_ref_val(parent)
-					.."["..dump_or_ref_val(k).."] = "
-					..dump_or_ref_val(v)
-			end
+	elseif first=="N" then
+		local num = tonumber(rest)
+		if num then
+			return num
+		else
+			file:close()
+			error("Unable to parse number: '"..rest.."'!")
 		end
-	end
-
-	mark_multiple_occurences(x)
-	local top_level = dump_or_ref_val(x)
-	dump_nest_points()
-
-	if next(local_defs) then
-		return "local _ = {}\n"
-			..table.concat(local_defs, "\n")
-			.."\nreturn "..top_level
+	elseif first=="B" then
+		if rest=="0" then
+			return false
+		elseif rest=="1" then
+			return true
+		else
+			file:close()
+			error("Unable to parse boolean: '"..rest.."'!")
+		end
+	elseif first=="S" then
+		return unescape_chars(rest)
 	else
-		return "return "..top_level
+		file:close()
+		error("Unknown literal type '"..first.."' for literal '"..str.."'!")
 	end
 end
 
--- Deserialization
+function unescape_chars(str) --TODO
+	local rstr = string.gsub(str, "&:", ":")
+	rstr = string.gsub(rstr, "&n", "\n")
+	rstr = string.gsub(rstr, "&r", "\r")
+	rstr = string.gsub(rstr, "&&", "&")
+	return rstr
+end
 
-local env = {
-	loadstring = loadstring,
-}
+------
 
-local safe_env = {
-	loadstring = function() end,
+--[[
+config = {
+	skip_empty_tables = false	-- if true, does not store empty tables
+								-- On next read, keys that mapped to empty tables resolve to nil
 }
+]]
+
+-- Writes the passed table into the passed file descriptor, and closes the file
+local function write_to_fd(root_table, file, config)
+	file:write("LUA_SER v=2\n")
+	write_table(root_table, file, config)
+	file:write("E\nEND_SER\n")
+	file:close()
+end
 
-function core.deserialize(str, safe)
-	if type(str) ~= "string" then
-		return nil, "Cannot deserialize type '"..type(str)
-			.."'. Argument must be a string."
+-- Reads the file contents from the passed file descriptor and returns the table on success
+-- Throws errors when something is wrong. Closes the file.
+-- config: see above
+local function read_from_fd(file)
+	local first_line = file:read("*line")
+	if not string.match(first_line, "LUA_SER v=[12]") then
+		file:close()
+		error("Expected header, got '"..first_line.."' instead!")
 	end
-	if str:byte(1) == 0x1B then
-		return nil, "Bytecode prohibited"
+	local t = {}
+	read_table(t, file)
+	local last_line = file:read("*line")
+	file:close()
+	if last_line ~= "END_SER" then
+		error("Missing END_SER, got '"..last_line.."' instead!")
 	end
-	local f, err = loadstring(str)
-	if not f then return nil, err end
-	setfenv(f, safe and safe_env or env)
+	return t
+end
 
-	local good, data = pcall(f)
-	if good then
-		return data
-	else
-		return nil, data
+-- Opens the passed filename and serializes root_table into it
+-- config: see above
+function write_to_file(root_table, filename, config)
+	-- try opening the file
+	local file, err = io.open(filename, "wb")
+	if not file then
+		error("Failed opening file '"..filename.."' for write:\n"..err)
 	end
+	
+	write_to_fd(root_table, file, config)
+	return true
 end
 
+-- Opens the passed filename, and returns its deserialized contents
+function read_from_file(filename)
+	-- try opening the file
+	local file, err = io.open(filename, "rb")
+	if not file then
+		error("Failed opening file '"..filename.."' for read:\n"..err)
+	end
+	
+	return read_from_fd(file)
+end
 
--- Unit tests
-local test_in = {cat={sound="nyan", speed=400}, dog={sound="woof"}}
-local test_out = core.deserialize(core.serialize(test_in))
+--[[ simple unit test
+local testtable = {
+	key = "value",
+	[1] = "eins",
+	[true] = {
+		a = "b",
+		c = false,
+	},
+	["es:cape1"] = "foo:bar",
+	["es&ca\npe2"] = "baz&bam\nbim",
+	["es&&ca&\npe3"] = "baz&&bam&\nbim",
+	["es&:cape4"] = "foo\n:bar"
+}
+local config = {}
+--write_to_file(testtable, "test_out", config)
+local t = read_from_file("test_out")
+write_to_file(t, "test_out_2", config)
+local t2 = read_from_file("test_out_2")
+write_to_file(t2, "test_out_3", config)
 
-assert(test_in.cat.sound == test_out.cat.sound)
-assert(test_in.cat.speed == test_out.cat.speed)
-assert(test_in.dog.sound == test_out.dog.sound)
+-- test_out_2 and test_out_3 should be equal
 
-test_in = {escape_chars="\n\r\t\v\\\"\'", non_european="θשׁ٩∂"}
-test_out = core.deserialize(core.serialize(test_in))
-assert(test_in.escape_chars == test_out.escape_chars)
-assert(test_in.non_european == test_out.non_european)
+--]]
 
+
+return {
+	read_from_fd = read_from_fd,
+	write_to_fd = write_to_fd,
+	read_from_file = read_from_file,
+	write_to_file = write_to_file,
+}