From faf8d88167f065ba3b2829badef14a31c4574971 Mon Sep 17 00:00:00 2001 From: Gael-de-Sailly Date: Wed, 12 May 2021 11:17:41 +0200 Subject: Updated to be compatible with new advtrains file structure May be hacky, but works. --- main.lua | 27 ++-- nodedb.lua | 83 +++++++++++- routes.lua | 11 +- serialize.lua | 425 +++++++++++++++++++++++++++++++++------------------------- 4 files changed, 338 insertions(+), 208 deletions(-) diff --git a/main.lua b/main.lua index 8ff86cd..93dadd9 100644 --- a/main.lua +++ b/main.lua @@ -85,7 +85,7 @@ function atdump(t, intend) end dofile("vector.lua") -dofile("serialize.lua") +local serialize = dofile("serialize.lua") dofile("helpers.lua") dofile("tracks.lua") dofile("track_defs.lua") @@ -128,22 +128,25 @@ end datapath, mappath, no_trains, worldimage = parse_args(arg) -- Load saves -local file, err = io.open(datapath.."advtrains_trains", "r") -local tbl = minetest.deserialize(file:read("*a")) +local tbl = serialize.read_from_file(datapath.."advtrains_core.ls") +--local file, err = io.open(datapath.."advtrains_trains", "r") +--local tbl = minetest.deserialize(file:read("*a")) if type(tbl) ~= "table" then error("Trains file: not a table") end -advtrains.trains = tbl -file:close() +advtrains.trains = tbl.trains +--file:close() --ndb contains the defs, while ndb2 is the actual contents -file, err = io.open(datapath.."advtrains_ndb", "r") -tbl = minetest.deserialize(file:read("*a")) -if type(tbl) ~= "table" then - error("Node database file: not a table") -end -advtrains.ndb.load_data(tbl) -file:close() +dofile("nodedb.lua") +local file, err = io.open(datapath.."advtrains_ndb4.ls", "r") +--tbl = minetest.deserialize(file:read("*a")) +--if type(tbl) ~= "table" then + --error("Node database file: not a table") +--end +--advtrains.ndb.load_data(tbl) +advtrains.ndb.load_callback(file) +--file:close() -- open svg file diff --git a/nodedb.lua b/nodedb.lua index de582f0..6f852d6 100644 --- a/nodedb.lua +++ b/nodedb.lua @@ -31,6 +31,7 @@ local ndb={} --local variables for performance local ndb_nodeids={} local ndb_nodes={} +local ndb_ver local function ndbget(x,y,z) local ny=ndb_nodes[y] @@ -52,22 +53,39 @@ local function ndbset(x,y,z,v) ndb_nodes[y][x][z]=v end +-- load/save -local path="advtrains_ndb2" ---load +local path_pre_v4=datapath.."advtrains_ndb2" +--load pre_v4 format --nodeids get loaded by advtrains init.lua and passed here -function ndb.load_data(data) +function ndb.load_data_pre_v4(data) + print("nodedb: Loading pre v4 format") + ndb_nodeids = data and data.nodeids or {} - local file, err = io.open(datapath..path, "rb") + ndb_ver = data and data.ver or 0 + if ndb_ver < 1 then + for k,v in pairs(ndb_nodeids) do + if v == "advtrains:dtrack_xing4590_st" then + cidDepr = k + elseif v == "advtrains:dtrack_xing90plusx_45l" then + cidNew = k + end + end + end + local file, err = io.open(path_pre_v4, "rb") if not file then print("Couldn't load the node database: ", err or "Unknown Error") else + -- Note: code duplication because of weird coordinate order in ndb2 format (z,y,x) local cnt=0 local hst_z=file:read(2) local hst_y=file:read(2) local hst_x=file:read(2) local cid=file:read(2) while hst_z and hst_y and hst_x and cid and #hst_z==2 and #hst_y==2 and #hst_x==2 and #cid==2 do + if (ndb_ver < 1 and cid == cidDepr) then + cid = cidNew + end ndbset(bytes_to_int(hst_x), bytes_to_int(hst_y), bytes_to_int(hst_z), bytes_to_int(cid)) cnt=cnt+1 hst_z=file:read(2) @@ -75,12 +93,66 @@ function ndb.load_data(data) hst_x=file:read(2) cid=file:read(2) end - print("nodedb: read", cnt, "nodes.") + print("nodedb (ndb2 format): read", cnt, "nodes.") ndb_nodes_total = cnt file:close() end + ndb_ver = 1 +end + +-- the new ndb file format is backported from cellworld, and stores the cids also in the ndb file. +-- These functions have the form of a serialize_lib atomic load/save callback and are called from avt_save/avt_load. +function ndb.load_callback(file) + -- read version + local vers_byte = file:read(1) + local version = string.byte(vers_byte) + if version~=1 then + file:close() + error("Doesn't support v4 nodedb file of version "..version) + end + + -- read cid mappings + local nstr_byte = file:read(2) + local nstr = bytes_to_int(nstr_byte) + for i = 1,nstr do + local stid_byte = file:read(2) + local stid = bytes_to_int(stid_byte) + local stna = file:read("*l") + --atdebug("content id:", stid, "->", stna) + ndb_nodeids[stid] = stna + end + print("[nodedb] read", nstr, "node content ids.") + + -- read nodes + local cnt=0 + local hst_x=file:read(2) + local hst_y=file:read(2) + local hst_z=file:read(2) + local cid=file:read(2) + local cidi + while hst_z and hst_y and hst_x and cid and #hst_z==2 and #hst_y==2 and #hst_x==2 and #cid==2 do + cidi = bytes_to_int(cid) + -- prevent file corruption already here + if not ndb_nodeids[u14b(cidi)] then + -- clear the ndb data, to reinitialize it + -- in strict loading mode, doesn't matter as starting will be interrupted anyway + ndb_nodeids = {} + ndb_nodes = {} + error("NDB file is corrupted (found entry with invalid cid)") + end + ndbset(bytes_to_int(hst_x), bytes_to_int(hst_y), bytes_to_int(hst_z), cidi) + cnt=cnt+1 + hst_x=file:read(2) + hst_y=file:read(2) + hst_z=file:read(2) + cid=file:read(2) + end + print("[nodedb] read", cnt, "nodes.") + ndb_nodes_total = cnt + file:close() end + --function to get node. track database is not helpful here. function ndb.get_node_or_nil(pos) -- FIX for bug found on linuxworks server: @@ -124,6 +196,7 @@ end --false if it's not a rail or the train does not drive on this rail, but it is loaded or --nil if the node is neither loaded nor in trackdb --the distraction between false and nil will be needed only in special cases.(train initpos) +advtrains = advtrains or {} function advtrains.get_rail_info_at(pos) local rdp=advtrains.round_vector_floor_y(pos) diff --git a/routes.lua b/routes.lua index 04b96fe..e828b67 100755 --- a/routes.lua +++ b/routes.lua @@ -8,7 +8,7 @@ math.hypot = function(a,b) return math.sqrt(a*a + b*b) end function attrans(str) return str end dofile("vector.lua") -dofile("serialize.lua") +local serialize = dofile("serialize.lua") dofile("helpers.lua") @@ -72,10 +72,11 @@ function ars_to_text(arstab) end -local file = io.open(datapath.."advtrains_interlocking_tcbs", "r") -local tbl = minetest.deserialize(file:read("*a")) -advtrains.tcbs = tbl -file:close() +--local file = io.open(datapath.."advtrains_interlocking_tcbs", "r") +--local tbl = minetest.deserialize(file:read("*a")) +local tbl = serialize.read_from_file(datapath.."advtrains_interlocking") +advtrains.tcbs = tbl.tcbs +--file:close() local jsonfile = io.open(datapath.."signals.json", "w") diff --git a/serialize.lua b/serialize.lua index 692ddd5..4b1ebb7 100755 --- a/serialize.lua +++ b/serialize.lua @@ -1,221 +1,274 @@ ---- Lua module to serialize values as Lua code. --- From: https://github.com/fab13n/metalua/blob/no-dll/src/lib/serialize.lua --- License: MIT --- @copyright 2006-2997 Fabien Fleutot --- @author Fabien Fleutot --- @author ShadowNinja --------------------------------------------------------------------------------- - ---- Serialize an object into a source code string. This string, when passed as --- an argument to deserialize(), returns an object structurally identical to --- the original one. The following are currently supported: --- * Booleans, numbers, strings, and nil. --- * Functions; uses interpreter-dependent (and sometimes platform-dependent) bytecode! --- * Tables; they can cantain multiple references and can be recursive, but metatables aren't saved. --- This works in two phases: --- 1. Recursively find and record multiple references and recursion. --- 2. Recursively dump the value into a string. --- @param x Value to serialize (nil is allowed). --- @return load()able string containing the value. -function core.serialize(x) - local local_index = 1 -- Top index of the "_" local table in the dump - -- table->nil/1/2 set of tables seen. - -- nil = not seen, 1 = seen once, 2 = seen multiple times. - local seen = {} - - -- nest_points are places where a table appears within itself, directly - -- or not. For instance, all of these chunks create nest points in - -- table x: "x = {}; x[x] = 1", "x = {}; x[1] = x", - -- "x = {}; x[1] = {y = {x}}". - -- To handle those, two tables are used by mark_nest_point: - -- * nested - Transient set of tables being currently traversed. - -- Used for detecting nested tables. - -- * nest_points - parent->{key=value, ...} table cantaining the nested - -- keys and values in the parent. They're all dumped after all the - -- other table operations have been performed. - -- - -- mark_nest_point(p, k, v) fills nest_points with information required - -- to remember that key/value (k, v) creates a nest point in table - -- parent. It also marks "parent" and the nested item(s) as occuring - -- multiple times, since several references to it will be required in - -- order to patch the nest points. - local nest_points = {} - local nested = {} - local function mark_nest_point(parent, k, v) - local nk, nv = nested[k], nested[v] - local np = nest_points[parent] - if not np then - np = {} - nest_points[parent] = np - end - np[k] = v - seen[parent] = 2 - if nk then seen[k] = 2 end - if nv then seen[v] = 2 end +-- serialize.lua +-- Lua-conformant library file that has no minetest dependencies +-- Contains the serialization and deserialization routines + +--[[ +Version history: +1 - initial +2 - also escaping CR character as &r + +Structure of entry: +[keytype][key]:[valuetype][val] +Types: + B - bool + -> 0=false, 1=true + S - string + -> see below + N - number + -> thing compatible with tonumber() +Table: +[keytype][key]:T +... content is nested in table until the matching +E + +example: +LUA_SER v=2 { +Skey:Svalue key = "value", +N1:Seins [1] = "eins", +B1:T [true] = { +Sa:Sb a = "b", +Sc:B0 c = false, +E } +E } + +String representations: +In strings the following characters are escaped by & +'&' -> '&&' +(line break) -> '&n' +(CR) -> '&r' +':' -> '&:' +All other characters are unchanged as they bear no special meaning. +]] + +local write_table, literal_to_string, escape_chars, table_is_empty + +function table_is_empty(t) + for _,_ in pairs(t) do + return false end + return true +end - -- First phase, list the tables and functions which appear more than - -- once in x. - local function mark_multiple_occurences(x) - local tp = type(x) - if tp ~= "table" and tp ~= "function" then - -- No identity (comparison is done by value, not by instance) - return +function write_table(t, file, config) + local ks, vs, writeit, istable + for key, value in pairs(t) do + ks = value_to_string(key, false) + writeit = true + istable = type(value)=="table" + + if istable then + vs = "T" + if config and config.skip_empty_tables then + writeit = not table_is_empty(value) + end + else + vs = value_to_string(value, true) end - if seen[x] == 1 then - seen[x] = 2 - elseif seen[x] ~= 2 then - seen[x] = 1 + + if writeit then + file:write(ks..":"..vs.."\n") + + if istable then + write_table(value, file, config) + file:write("E\n") + end end + end +end - if tp == "table" then - nested[x] = true - for k, v in pairs(x) do - if nested[k] or nested[v] then - mark_nest_point(x, k, v) - else - mark_multiple_occurences(k) - mark_multiple_occurences(v) - end - end - nested[x] = nil +function value_to_string(t) + if type(t)=="table" then + file:close() + error("Can not serialize a table in the key position!") + elseif type(t)=="boolean" then + if t then + return "B1" + else + return "B0" end + elseif type(t)=="number" then + return "N"..t + elseif type(t)=="string" then + return "S"..escape_chars(t) + else + --error("Can not serialize '"..type(t).."' type!") + return "S" end + return str +end - local dumped = {} -- object->varname set - local local_defs = {} -- Dumped local definitions as source code lines +function escape_chars(str) + local rstr = string.gsub(str, "&", "&&") + rstr = string.gsub(rstr, ":", "&:") + rstr = string.gsub(rstr, "\r", "&r") + rstr = string.gsub(rstr, "\n", "&n") + return rstr +end + +------ - -- Mutually recursive local functions: - local dump_val, dump_or_ref_val +local read_table, string_to_value, unescape_chars - -- If x occurs multiple times, dump the local variable rather than - -- the value. If it's the first time it's dumped, also dump the - -- content in local_defs. - function dump_or_ref_val(x) - if seen[x] ~= 2 then - return dump_val(x) +function read_table(t, file) + local line, ks, vs, kv, vv, vt + while true do + line = file:read("*l") + if not line then + file:close() + error("Unexpected EOF or read error!") + end + + if line=="E" then + -- done with this table + return + end + ks, vs = string.match(line, "^(.*[^&]):(.+)$") + if not ks or not vs then + file:close() + error("Unable to parse line: '"..line.."'!") end - local var = dumped[x] - if var then -- Already referenced - return var + kv = string_to_value(ks) + vv, vt = string_to_value(vs, true) + if vt then + read_table(vv, file) end - -- First occurence, create and register reference - local val = dump_val(x) - local i = local_index - local_index = local_index + 1 - var = "_["..i.."]" - local_defs[#local_defs + 1] = var.." = "..val - dumped[x] = var - return var + -- put read value in table + t[kv] = vv end +end - -- Second phase. Dump the object; subparts occuring multiple times - -- are dumped in local variables which can be referenced multiple - -- times. Care is taken to dump local vars in a sensible order. - function dump_val(x) - local tp = type(x) - if x == nil then return "nil" - elseif tp == "string" then return string.format("%q", x) - elseif tp == "boolean" then return x and "true" or "false" - elseif tp == "function" then - return string.format("loadstring(%q)", string.dump(x)) - elseif tp == "number" then - -- Serialize integers with string.format to prevent - -- scientific notation, which doesn't preserve - -- precision and breaks things like node position - -- hashes. Serialize floats normally. - if math.floor(x) == x then - return string.format("%d", x) - else - return tostring(x) - end - elseif tp == "table" then - local vals = {} - local idx_dumped = {} - local np = nest_points[x] - for i, v in ipairs(x) do - if not np or not np[i] then - vals[#vals + 1] = dump_or_ref_val(v) - end - idx_dumped[i] = true - end - for k, v in pairs(x) do - if (not np or not np[k]) and - not idx_dumped[k] then - vals[#vals + 1] = "["..dump_or_ref_val(k).."] = " - ..dump_or_ref_val(v) - end - end - return "{"..table.concat(vals, ", ").."}" +-- returns: value, is_table +function string_to_value(str, table_allow) + local first = string.sub(str, 1,1) + local rest = string.sub(str, 2) + if first=="T" then + if table_allow then + return {}, true else - error("Can't serialize data of type "..tp) + file:close() + error("Table not allowed in key component!") end - end - - local function dump_nest_points() - for parent, vals in pairs(nest_points) do - for k, v in pairs(vals) do - local_defs[#local_defs + 1] = dump_or_ref_val(parent) - .."["..dump_or_ref_val(k).."] = " - ..dump_or_ref_val(v) - end + elseif first=="N" then + local num = tonumber(rest) + if num then + return num + else + file:close() + error("Unable to parse number: '"..rest.."'!") end - end - - mark_multiple_occurences(x) - local top_level = dump_or_ref_val(x) - dump_nest_points() - - if next(local_defs) then - return "local _ = {}\n" - ..table.concat(local_defs, "\n") - .."\nreturn "..top_level + elseif first=="B" then + if rest=="0" then + return false + elseif rest=="1" then + return true + else + file:close() + error("Unable to parse boolean: '"..rest.."'!") + end + elseif first=="S" then + return unescape_chars(rest) else - return "return "..top_level + file:close() + error("Unknown literal type '"..first.."' for literal '"..str.."'!") end end --- Deserialization +function unescape_chars(str) --TODO + local rstr = string.gsub(str, "&:", ":") + rstr = string.gsub(rstr, "&n", "\n") + rstr = string.gsub(rstr, "&r", "\r") + rstr = string.gsub(rstr, "&&", "&") + return rstr +end -local env = { - loadstring = loadstring, -} +------ -local safe_env = { - loadstring = function() end, +--[[ +config = { + skip_empty_tables = false -- if true, does not store empty tables + -- On next read, keys that mapped to empty tables resolve to nil } +]] + +-- Writes the passed table into the passed file descriptor, and closes the file +local function write_to_fd(root_table, file, config) + file:write("LUA_SER v=2\n") + write_table(root_table, file, config) + file:write("E\nEND_SER\n") + file:close() +end -function core.deserialize(str, safe) - if type(str) ~= "string" then - return nil, "Cannot deserialize type '"..type(str) - .."'. Argument must be a string." +-- Reads the file contents from the passed file descriptor and returns the table on success +-- Throws errors when something is wrong. Closes the file. +-- config: see above +local function read_from_fd(file) + local first_line = file:read("*line") + if not string.match(first_line, "LUA_SER v=[12]") then + file:close() + error("Expected header, got '"..first_line.."' instead!") end - if str:byte(1) == 0x1B then - return nil, "Bytecode prohibited" + local t = {} + read_table(t, file) + local last_line = file:read("*line") + file:close() + if last_line ~= "END_SER" then + error("Missing END_SER, got '"..last_line.."' instead!") end - local f, err = loadstring(str) - if not f then return nil, err end - setfenv(f, safe and safe_env or env) + return t +end - local good, data = pcall(f) - if good then - return data - else - return nil, data +-- Opens the passed filename and serializes root_table into it +-- config: see above +function write_to_file(root_table, filename, config) + -- try opening the file + local file, err = io.open(filename, "wb") + if not file then + error("Failed opening file '"..filename.."' for write:\n"..err) end + + write_to_fd(root_table, file, config) + return true end +-- Opens the passed filename, and returns its deserialized contents +function read_from_file(filename) + -- try opening the file + local file, err = io.open(filename, "rb") + if not file then + error("Failed opening file '"..filename.."' for read:\n"..err) + end + + return read_from_fd(file) +end --- Unit tests -local test_in = {cat={sound="nyan", speed=400}, dog={sound="woof"}} -local test_out = core.deserialize(core.serialize(test_in)) +--[[ simple unit test +local testtable = { + key = "value", + [1] = "eins", + [true] = { + a = "b", + c = false, + }, + ["es:cape1"] = "foo:bar", + ["es&ca\npe2"] = "baz&bam\nbim", + ["es&&ca&\npe3"] = "baz&&bam&\nbim", + ["es&:cape4"] = "foo\n:bar" +} +local config = {} +--write_to_file(testtable, "test_out", config) +local t = read_from_file("test_out") +write_to_file(t, "test_out_2", config) +local t2 = read_from_file("test_out_2") +write_to_file(t2, "test_out_3", config) -assert(test_in.cat.sound == test_out.cat.sound) -assert(test_in.cat.speed == test_out.cat.speed) -assert(test_in.dog.sound == test_out.dog.sound) +-- test_out_2 and test_out_3 should be equal -test_in = {escape_chars="\n\r\t\v\\\"\'", non_european="θשׁ٩∂"} -test_out = core.deserialize(core.serialize(test_in)) -assert(test_in.escape_chars == test_out.escape_chars) -assert(test_in.non_european == test_out.non_european) +--]] + +return { + read_from_fd = read_from_fd, + write_to_fd = write_to_fd, + read_from_file = read_from_file, + write_to_file = write_to_file, +} -- cgit v1.2.3