Use JSON for favorites, move server list code to Lua (#10085)

Co-authored-by: sfan5 <sfan5@live.de>

7 files changed, 368 insertions, 109 deletions

diff --git a/builtin/mainmenu/common.lua b/builtin/mainmenu/common.lua
index 2bd8aa8a5..01f9a30b9 100644
--- a/builtin/mainmenu/common.lua
+++ b/builtin/mainmenu/common.lua
@@ -62,24 +62,6 @@ function image_column(tooltip, flagname)
 		"5=" .. core.formspec_escape(defaulttexturedir .. "server_ping_1.png")
 end
 
---------------------------------------------------------------------------------
-function order_favorite_list(list)
-	local res = {}
-	--orders the favorite list after support
-	for i = 1, #list do
-		local fav = list[i]
-		if is_server_protocol_compat(fav.proto_min, fav.proto_max) then
-			res[#res + 1] = fav
-		end
-	end
-	for i = 1, #list do
-		local fav = list[i]
-		if not is_server_protocol_compat(fav.proto_min, fav.proto_max) then
-			res[#res + 1] = fav
-		end
-	end
-	return res
-end
 
 --------------------------------------------------------------------------------
 function render_serverlist_row(spec, is_favorite)
@@ -227,41 +209,6 @@ function menu_handle_key_up_down(fields, textlist, settingname)
 end
 
 --------------------------------------------------------------------------------
-function asyncOnlineFavourites()
-	if not menudata.public_known then
-		menudata.public_known = {{
-			name = fgettext("Loading..."),
-			description = fgettext_ne("Try reenabling public serverlist and check your internet connection.")
-		}}
-	end
-	menudata.favorites = menudata.public_known
-	menudata.favorites_is_public = true
-
-	if not menudata.public_downloading then
-		menudata.public_downloading = true
-	else
-		return
-	end
-
-	core.handle_async(
-		function(param)
-			return core.get_favorites("online")
-		end,
-		nil,
-		function(result)
-			menudata.public_downloading = nil
-			local favs = order_favorite_list(result)
-			if favs[1] then
-				menudata.public_known = favs
-				menudata.favorites = menudata.public_known
-				menudata.favorites_is_public = true
-			end
-			core.event_handler("Refresh")
-		end
-	)
-end
-
---------------------------------------------------------------------------------
 function text2textlist(xpos, ypos, width, height, tl_name, textlen, text, transparency)
 	local textlines = core.wrap_text(text, textlen, true)
 	local retval = "textlist[" .. xpos .. "," .. ypos .. ";" .. width ..
diff --git a/builtin/mainmenu/init.lua b/builtin/mainmenu/init.lua
index 656d1d149..45089c7c9 100644
--- a/builtin/mainmenu/init.lua
+++ b/builtin/mainmenu/init.lua
@@ -34,6 +34,7 @@ dofile(basepath .. "fstk" .. DIR_DELIM .. "ui.lua")
 dofile(menupath .. DIR_DELIM .. "async_event.lua")
 dofile(menupath .. DIR_DELIM .. "common.lua")
 dofile(menupath .. DIR_DELIM .. "pkgmgr.lua")
+dofile(menupath .. DIR_DELIM .. "serverlistmgr.lua")
 dofile(menupath .. DIR_DELIM .. "textures.lua")
 
 dofile(menupath .. DIR_DELIM .. "dlg_config_world.lua")
diff --git a/builtin/mainmenu/serverlistmgr.lua b/builtin/mainmenu/serverlistmgr.lua
new file mode 100644
index 000000000..d98736e54
--- /dev/null
+++ b/builtin/mainmenu/serverlistmgr.lua
@@ -0,0 +1,241 @@
+--Minetest
+--Copyright (C) 2020 rubenwardy
+--
+--This program is free software; you can redistribute it and/or modify
+--it under the terms of the GNU Lesser General Public License as published by
+--the Free Software Foundation; either version 2.1 of the License, or
+--(at your option) any later version.
+--
+--This program is distributed in the hope that it will be useful,
+--but WITHOUT ANY WARRANTY; without even the implied warranty of
+--MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+--GNU Lesser General Public License for more details.
+--
+--You should have received a copy of the GNU Lesser General Public License along
+--with this program; if not, write to the Free Software Foundation, Inc.,
+--51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+serverlistmgr = {}
+
+--------------------------------------------------------------------------------
+local function order_server_list(list)
+	local res = {}
+	--orders the favorite list after support
+	for i = 1, #list do
+		local fav = list[i]
+		if is_server_protocol_compat(fav.proto_min, fav.proto_max) then
+			res[#res + 1] = fav
+		end
+	end
+	for i = 1, #list do
+		local fav = list[i]
+		if not is_server_protocol_compat(fav.proto_min, fav.proto_max) then
+			res[#res + 1] = fav
+		end
+	end
+	return res
+end
+
+local public_downloading = false
+
+--------------------------------------------------------------------------------
+function serverlistmgr.sync()
+	if not serverlistmgr.servers then
+		serverlistmgr.servers = {{
+			name = fgettext("Loading..."),
+			description = fgettext_ne("Try reenabling public serverlist and check your internet connection.")
+		}}
+	end
+
+	if public_downloading then
+		return
+	end
+	public_downloading = true
+
+	core.handle_async(
+		function(param)
+			local http = core.get_http_api()
+			local url = ("%s/list?proto_version_min=%d&proto_version_max=%d"):format(
+				core.settings:get("serverlist_url"),
+				core.get_min_supp_proto(),
+				core.get_max_supp_proto())
+
+			local response = http.fetch_sync({ url = url })
+			if not response.succeeded then
+				return {}
+			end
+
+			local retval = core.parse_json(response.data)
+			return retval and retval.list or {}
+		end,
+		nil,
+		function(result)
+			public_downloading = nil
+			local favs = order_server_list(result)
+			if favs[1] then
+				serverlistmgr.servers = favs
+			end
+			core.event_handler("Refresh")
+		end
+	)
+end
+
+--------------------------------------------------------------------------------
+local function get_favorites_path()
+	local base = core.get_user_path() .. DIR_DELIM .. "client" .. DIR_DELIM .. "serverlist" .. DIR_DELIM
+	return base .. core.settings:get("serverlist_file")
+end
+
+--------------------------------------------------------------------------------
+local function save_favorites(favorites)
+	local filename = core.settings:get("serverlist_file")
+	-- If setting specifies legacy format change the filename to the new one
+	if filename:sub(#filename - 3):lower() == ".txt" then
+		core.settings:set("serverlist_file", filename:sub(1, #filename - 4) .. ".json")
+	end
+
+	local path = get_favorites_path()
+	core.create_dir(path)
+	core.safe_file_write(path, core.write_json(favorites))
+end
+
+--------------------------------------------------------------------------------
+function serverlistmgr.read_legacy_favorites(path)
+	local file = io.open(path, "r")
+	if not file then
+		return nil
+	end
+
+	local lines = {}
+	for line in file:lines() do
+		lines[#lines + 1] = line
+	end
+	file:close()
+
+	local favorites = {}
+
+	local i = 1
+	while i < #lines do
+		local function pop()
+			local line = lines[i]
+			i = i + 1
+			return line and line:trim()
+		end
+
+		if pop():lower() == "[server]" then
+			local name = pop()
+			local address = pop()
+			local port = tonumber(pop())
+			local description = pop()
+
+			if name == "" then
+				name = nil
+			end
+
+			if description == "" then
+				description = nil
+			end
+
+			if not address or #address < 3 then
+				core.log("warning", "Malformed favorites file, missing address at line " .. i)
+			elseif not port or port < 1 or port > 65535 then
+				core.log("warning", "Malformed favorites file, missing port at line " .. i)
+			elseif (name and name:upper() == "[SERVER]") or
+					(address and address:upper() == "[SERVER]") or
+					(description and description:upper() == "[SERVER]") then
+				core.log("warning", "Potentially malformed favorites file, overran at line " .. i)
+			else
+				favorites[#favorites + 1] = {
+					name = name,
+					address = address,
+					port = port,
+					description = description
+				}
+			end
+		end
+	end
+
+	return favorites
+end
+
+--------------------------------------------------------------------------------
+local function read_favorites()
+	local path = get_favorites_path()
+
+	-- If new format configured fall back to reading the legacy file
+	if path:sub(#path - 4):lower() == ".json" then
+		local file = io.open(path, "r")
+		if file then
+			local json = file:read("*all")
+			file:close()
+			return core.parse_json(json)
+		end
+
+		path = path:sub(1, #path - 5) .. ".txt"
+	end
+
+	local favs = serverlistmgr.read_legacy_favorites(path)
+	if favs then
+		save_favorites(favs)
+		os.remove(path)
+	end
+	return favs
+end
+
+--------------------------------------------------------------------------------
+local function delete_favorite(favorites, del_favorite)
+	for i=1, #favorites do
+		local fav = favorites[i]
+
+		if fav.address == del_favorite.address and fav.port == del_favorite.port then
+			table.remove(favorites, i)
+			return
+		end
+	end
+end
+
+--------------------------------------------------------------------------------
+function serverlistmgr.get_favorites()
+	if serverlistmgr.favorites then
+		return serverlistmgr.favorites
+	end
+
+	serverlistmgr.favorites = {}
+
+	-- Add favorites, removing duplicates
+	local seen = {}
+	for _, fav in ipairs(read_favorites() or {}) do
+		local key = ("%s:%d"):format(fav.address:lower(), fav.port)
+		if not seen[key] then
+			seen[key] = true
+			serverlistmgr.favorites[#serverlistmgr.favorites + 1] = fav
+		end
+	end
+
+	return serverlistmgr.favorites
+end
+
+--------------------------------------------------------------------------------
+function serverlistmgr.add_favorite(new_favorite)
+	assert(type(new_favorite.port) == "number")
+
+	-- Whitelist favorite keys
+	new_favorite = {
+		name = new_favorite.name,
+		address = new_favorite.address,
+		port = new_favorite.port,
+		description = new_favorite.description,
+	}
+
+	local favorites = serverlistmgr.get_favorites()
+	delete_favorite(favorites, new_favorite)
+	table.insert(favorites, 1, new_favorite)
+	save_favorites(favorites)
+end
+
+--------------------------------------------------------------------------------
+function serverlistmgr.delete_favorite(del_favorite)
+	local favorites = serverlistmgr.get_favorites()
+	delete_favorite(favorites, del_favorite)
+	save_favorites(favorites)
+end
diff --git a/builtin/mainmenu/tab_online.lua b/builtin/mainmenu/tab_online.lua
index 8f1341161..e6748ed88 100644
--- a/builtin/mainmenu/tab_online.lua
+++ b/builtin/mainmenu/tab_online.lua
@@ -20,11 +20,11 @@ local function get_formspec(tabview, name, tabdata)
 	-- Update the cached supported proto info,
 	-- it may have changed after a change by the settings menu.
 	common_update_cached_supp_proto()
-	local fav_selected
+	local selected
 	if menudata.search_result then
-		fav_selected = menudata.search_result[tabdata.fav_selected]
+		selected = menudata.search_result[tabdata.selected]
 	else
-		fav_selected = menudata.favorites[tabdata.fav_selected]
+		selected = serverlistmgr.servers[tabdata.selected]
 	end
 
 	if not tabdata.search_for then
@@ -58,18 +58,18 @@ local function get_formspec(tabview, name, tabdata)
 		-- Connect
 		"button[9.88,4.9;2.3,1;btn_mp_connect;" .. fgettext("Connect") .. "]"
 
-	if tabdata.fav_selected and fav_selected then
+	if tabdata.selected and selected then
 		if gamedata.fav then
 			retval = retval .. "button[7.73,4.9;2.3,1;btn_delete_favorite;" ..
 				fgettext("Del. Favorite") .. "]"
 		end
-		if fav_selected.description then
+		if selected.description then
 			retval = retval .. "textarea[8.1,2.3;4.23,2.9;;;" ..
 				core.formspec_escape((gamedata.serverdescription or ""), true) .. "]"
 		end
 	end
 
-	--favourites
+	--favorites
 	retval = retval .. "tablecolumns[" ..
 		image_column(fgettext("Favorite"), "favorite") .. ";" ..
 		image_column(fgettext("Ping")) .. ",padding=0.25;" ..
@@ -83,13 +83,12 @@ local function get_formspec(tabview, name, tabdata)
 		image_column(fgettext("PvP enabled"), "pvp") .. ",padding=0.25;" ..
 		"color,span=1;" ..
 		"text,padding=1]" ..
-		"table[-0.15,0.6;7.75,5.15;favourites;"
+		"table[-0.15,0.6;7.75,5.15;favorites;"
 
 	if menudata.search_result then
+		local favs = serverlistmgr.get_favorites()
 		for i = 1, #menudata.search_result do
-			local favs = core.get_favorites("local")
 			local server = menudata.search_result[i]
-
 			for fav_id = 1, #favs do
 				if server.address == favs[fav_id].address and
 						server.port == favs[fav_id].port then
@@ -103,29 +102,30 @@ local function get_formspec(tabview, name, tabdata)
 
 			retval = retval .. render_serverlist_row(server, server.is_favorite)
 		end
-	elseif #menudata.favorites > 0 then
-		local favs = core.get_favorites("local")
+	elseif #serverlistmgr.servers > 0 then
+		local favs = serverlistmgr.get_favorites()
 		if #favs > 0 then
 			for i = 1, #favs do
-			for j = 1, #menudata.favorites do
-				if menudata.favorites[j].address == favs[i].address and
-						menudata.favorites[j].port == favs[i].port then
-					table.insert(menudata.favorites, i, table.remove(menudata.favorites, j))
+				for j = 1, #serverlistmgr.servers do
+					if serverlistmgr.servers[j].address == favs[i].address and
+							serverlistmgr.servers[j].port == favs[i].port then
+						table.insert(serverlistmgr.servers, i, table.remove(serverlistmgr.servers, j))
+					end
 				end
-			end
-				if favs[i].address ~= menudata.favorites[i].address then
-					table.insert(menudata.favorites, i, favs[i])
+				if favs[i].address ~= serverlistmgr.servers[i].address then
+					table.insert(serverlistmgr.servers, i, favs[i])
 				end
 			end
 		end
-		retval = retval .. render_serverlist_row(menudata.favorites[1], (#favs > 0))
-		for i = 2, #menudata.favorites do
-			retval = retval .. "," .. render_serverlist_row(menudata.favorites[i], (i <= #favs))
+
+		retval = retval .. render_serverlist_row(serverlistmgr.servers[1], (#favs > 0))
+		for i = 2, #serverlistmgr.servers do
+			retval = retval .. "," .. render_serverlist_row(serverlistmgr.servers[i], (i <= #favs))
 		end
 	end
 
-	if tabdata.fav_selected then
-		retval = retval .. ";" .. tabdata.fav_selected .. "]"
+	if tabdata.selected then
+		retval = retval .. ";" .. tabdata.selected .. "]"
 	else
 		retval = retval .. ";0]"
 	end
@@ -135,21 +135,20 @@ end
 
 --------------------------------------------------------------------------------
 local function main_button_handler(tabview, fields, name, tabdata)
-	local serverlist = menudata.search_result or menudata.favorites
+	local serverlist = menudata.search_result or serverlistmgr.servers
 
 	if fields.te_name then
 		gamedata.playername = fields.te_name
 		core.settings:set("name", fields.te_name)
 	end
 
-	if fields.favourites then
-		local event = core.explode_table_event(fields.favourites)
+	if fields.favorites then
+		local event = core.explode_table_event(fields.favorites)
 		local fav = serverlist[event.row]
 
 		if event.type == "DCL" then
 			if event.row <= #serverlist then
-				if menudata.favorites_is_public and
-						not is_server_protocol_compat_or_error(
+				if not is_server_protocol_compat_or_error(
 							fav.proto_min, fav.proto_max) then
 					return true
 				end
@@ -178,7 +177,7 @@ local function main_button_handler(tabview, fields, name, tabdata)
 		if event.type == "CHG" then
 			if event.row <= #serverlist then
 				gamedata.fav = false
-				local favs = core.get_favorites("local")
+				local favs = serverlistmgr.get_favorites()
 				local address = fav.address
 				local port    = fav.port
 				gamedata.serverdescription = fav.description
@@ -194,28 +193,28 @@ local function main_button_handler(tabview, fields, name, tabdata)
 					core.settings:set("address", address)
 					core.settings:set("remote_port", port)
 				end
-				tabdata.fav_selected = event.row
+				tabdata.selected = event.row
 			end
 			return true
 		end
 	end
 
 	if fields.key_up or fields.key_down then
-		local fav_idx = core.get_table_index("favourites")
+		local fav_idx = core.get_table_index("favorites")
 		local fav = serverlist[fav_idx]
 
 		if fav_idx then
 			if fields.key_up and fav_idx > 1 then
 				fav_idx = fav_idx - 1
-			elseif fields.key_down and fav_idx < #menudata.favorites then
+			elseif fields.key_down and fav_idx < #serverlistmgr.servers then
 				fav_idx = fav_idx + 1
 			end
 		else
 			fav_idx = 1
 		end
 
-		if not menudata.favorites or not fav then
-			tabdata.fav_selected = 0
+		if not serverlistmgr.servers or not fav then
+			tabdata.selected = 0
 			return true
 		end
 
@@ -227,17 +226,17 @@ local function main_button_handler(tabview, fields, name, tabdata)
 			core.settings:set("remote_port", port)
 		end
 
-		tabdata.fav_selected = fav_idx
+		tabdata.selected = fav_idx
 		return true
 	end
 
 	if fields.btn_delete_favorite then
-		local current_favourite = core.get_table_index("favourites")
-		if not current_favourite then return end
+		local current_favorite = core.get_table_index("favorites")
+		if not current_favorite then return end
 
-		core.delete_favorite(current_favourite)
-		asyncOnlineFavourites()
-		tabdata.fav_selected = nil
+		serverlistmgr.delete_favorite(serverlistmgr.servers[current_favorite])
+		serverlistmgr.sync()
+		tabdata.selected = nil
 
 		core.settings:set("address", "")
 		core.settings:set("remote_port", "30000")
@@ -251,11 +250,11 @@ local function main_button_handler(tabview, fields, name, tabdata)
 	end
 
 	if fields.btn_mp_search or fields.key_enter_field == "te_search" then
-		tabdata.fav_selected = 1
+		tabdata.selected = 1
 		local input = fields.te_search:lower()
 		tabdata.search_for = fields.te_search
 
-		if #menudata.favorites < 2 then
+		if #serverlistmgr.servers < 2 then
 			return true
 		end
 
@@ -275,8 +274,8 @@ local function main_button_handler(tabview, fields, name, tabdata)
 
 		-- Search the serverlist
 		local search_result = {}
-		for i = 1, #menudata.favorites do
-			local server = menudata.favorites[i]
+		for i = 1, #serverlistmgr.servers do
+			local server = serverlistmgr.servers[i]
 			local found = 0
 			for k = 1, #keywords do
 				local keyword = keywords[k]
@@ -293,7 +292,7 @@ local function main_button_handler(tabview, fields, name, tabdata)
 				end
 			end
 			if found > 0 then
-				local points = (#menudata.favorites - i) / 5 + found
+				local points = (#serverlistmgr.servers - i) / 5 + found
 				server.points = points
 				table.insert(search_result, server)
 			end
@@ -312,7 +311,7 @@ local function main_button_handler(tabview, fields, name, tabdata)
 	end
 
 	if fields.btn_mp_refresh then
-		asyncOnlineFavourites()
+		serverlistmgr.sync()
 		return true
 	end
 
@@ -321,30 +320,36 @@ local function main_button_handler(tabview, fields, name, tabdata)
 		gamedata.playername = fields.te_name
 		gamedata.password   = fields.te_pwd
 		gamedata.address    = fields.te_address
-		gamedata.port       = fields.te_port
+		gamedata.port       = tonumber(fields.te_port)
 		gamedata.selected_world = 0
-		local fav_idx = core.get_table_index("favourites")
+		local fav_idx = core.get_table_index("favorites")
 		local fav = serverlist[fav_idx]
 
 		if fav_idx and fav_idx <= #serverlist and
-				fav.address == fields.te_address and
-				fav.port    == fields.te_port then
+				fav.address == gamedata.address and
+				fav.port    == gamedata.port then
+
+			serverlistmgr.add_favorite(fav)
 
 			gamedata.servername        = fav.name
 			gamedata.serverdescription = fav.description
 
-			if menudata.favorites_is_public and
-					not is_server_protocol_compat_or_error(
+			if not is_server_protocol_compat_or_error(
 						fav.proto_min, fav.proto_max) then
 				return true
 			end
 		else
 			gamedata.servername        = ""
 			gamedata.serverdescription = ""
+
+			serverlistmgr.add_favorite({
+				address = gamedata.address,
+				port = gamedata.port,
+			})
 		end
 
-		core.settings:set("address",     fields.te_address)
-		core.settings:set("remote_port", fields.te_port)
+		core.settings:set("address",     gamedata.address)
+		core.settings:set("remote_port", gamedata.port)
 
 		core.start()
 		return true
@@ -354,7 +359,7 @@ end
 
 local function on_change(type, old_tab, new_tab)
 	if type == "LEAVE" then return end
-	asyncOnlineFavourites()
+	serverlistmgr.sync()
 end
 
 --------------------------------------------------------------------------------
diff --git a/builtin/mainmenu/tests/favorites_wellformed.txt b/builtin/mainmenu/tests/favorites_wellformed.txt
new file mode 100644
index 000000000..8b87b4398
--- /dev/null
+++ b/builtin/mainmenu/tests/favorites_wellformed.txt
@@ -0,0 +1,29 @@
+[server]
+
+127.0.0.1
+30000
+
+
+[server]
+
+localhost
+30000
+
+
+[server]
+
+vps.rubenwardy.com
+30001
+
+
+[server]
+
+gundul.ddnss.de
+39155
+
+
+[server]
+VanessaE's Dreambuilder creative Server
+daconcepts.com
+30000
+VanessaE's Dreambuilder creative-mode server. Lots of mods, whitelisted buckets.
diff --git a/builtin/mainmenu/tests/serverlistmgr_spec.lua b/builtin/mainmenu/tests/serverlistmgr_spec.lua
new file mode 100644
index 000000000..148e9b794
--- /dev/null
+++ b/builtin/mainmenu/tests/serverlistmgr_spec.lua
@@ -0,0 +1,36 @@
+_G.core = {}
+_G.unpack = table.unpack
+_G.serverlistmgr = {}
+
+dofile("builtin/common/misc_helpers.lua")
+dofile("builtin/mainmenu/serverlistmgr.lua")
+
+local base = "builtin/mainmenu/tests/"
+
+describe("legacy favorites", function()
+	it("loads well-formed correctly", function()
+		local favs = serverlistmgr.read_legacy_favorites(base .. "favorites_wellformed.txt")
+
+		local expected = {
+			{
+				address = "127.0.0.1",
+				port = 30000,
+			},
+
+			{ address = "localhost", port = 30000 },
+
+			{ address = "vps.rubenwardy.com", port = 30001 },
+
+			{ address = "gundul.ddnss.de", port = 39155 },
+
+			{
+				address = "daconcepts.com",
+				port = 30000,
+				name = "VanessaE's Dreambuilder creative Server",
+				description = "VanessaE's Dreambuilder creative-mode server. Lots of mods, whitelisted buckets."
+			},
+		}
+
+		assert.same(expected, favs)
+	end)
+end)
diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt
index 7e23b5641..21118134e 100644
--- a/builtin/settingtypes.txt
+++ b/builtin/settingtypes.txt
@@ -964,7 +964,7 @@ serverlist_url (Serverlist URL) string servers.minetest.net
 
 #    File in client/serverlist/ that contains your favorite servers displayed in the
 #    Multiplayer Tab.
-serverlist_file (Serverlist file) string favoriteservers.txt
+serverlist_file (Serverlist file) string favoriteservers.json
 
 #    Maximum size of the out chat queue.
 #    0 to disable queueing and -1 to make the queue size unlimited.