From 05238dc5aec332970a082b84d174240c4262e95d Mon Sep 17 00:00:00 2001
From: Markus Koch <markus@notsyncing.net>
Date: Mon, 27 Apr 2020 17:37:12 +0200
Subject: Add categories for markers

This commit obsoletes the many fetch scripts and combines them
into a single script, creating a single json, but with category
information.
---
 scripts/geojson/fetch.sh            | 30 -----------------------
 scripts/geojson/fetch_all_points.sh | 48 +++++++++++++++++++++++++++++++++++++
 scripts/geojson/fetch_cities.sh     | 10 --------
 scripts/geojson/fetch_shops.sh      | 10 --------
 scripts/geojson/fetch_single.sh     | 31 ------------------------
 scripts/geojson/fetch_stations.sh   | 10 --------
 6 files changed, 48 insertions(+), 91 deletions(-)
 delete mode 100755 scripts/geojson/fetch.sh
 create mode 100755 scripts/geojson/fetch_all_points.sh
 delete mode 100755 scripts/geojson/fetch_cities.sh
 delete mode 100755 scripts/geojson/fetch_shops.sh
 delete mode 100755 scripts/geojson/fetch_single.sh
 delete mode 100755 scripts/geojson/fetch_stations.sh

(limited to 'scripts')

diff --git a/scripts/geojson/fetch.sh b/scripts/geojson/fetch.sh
deleted file mode 100755
index bd86239..0000000
--- a/scripts/geojson/fetch.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-fetch_category () {
-    echo "Fetching $1" > /dev/stderr
-    json=`curl "https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:$1&cmlimit=max" 2>/dev/null`
-    cities=`echo "$json" | json_reformat | sed -n 's/.*"title":\s*"\([^"]*\).*/\1/p' | tr ' ' '_' | tr '\n' ' '`
-    num=`echo $cities | tr ' ' '\n' | wc -l`
-    i=0
-    echo "["
-    for city in $cities; do
-	let i++
-	echo -n -e "\rFetched $i/$num" > /dev/stderr
-	./fetch_single.sh "$city"
-    done
-    echo > /dev/stderr
-    echo "{}]"
-}
-
-JSONDIR=../../htdocs/geojson
-
-mkdir -p $JSONDIR
-
-
-fetch_category Shops > $JSONDIR/shops.json
-fetch_category City > $JSONDIR/cities.json
-fetch_category Stations > $JSONDIR/stations.json
-fetch_category Parks > $JSONDIR/parks.json
-fetch_category Libraries > $JSONDIR/libraries.json
-fetch_category "CW_Complexes" > $JSONDIR/cw_complexes.json
-fetch_category Courts > $JSONDIR/courts.json
-fetch_category Waterway > $JSONDIR/waterway.json
-fetch_category Train_Depots > $JSONDIR/depots.json
diff --git a/scripts/geojson/fetch_all_points.sh b/scripts/geojson/fetch_all_points.sh
new file mode 100755
index 0000000..81c46e0
--- /dev/null
+++ b/scripts/geojson/fetch_all_points.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+json=`curl -s 'https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=search&redirects=1&converttitles=1&srsearch=%7B%7BCo%7C&srlimit=max&srwhat=text&srprop=snippet'`
+data=`echo "$json" | json_reformat`;
+
+echo "[";
+
+IFS=$'\n';
+for line in $data; do
+	temp=`echo "$line" | sed -n 's/^\s*{\s*$/END/p'`
+	if [[ "$temp" == "END" ]]; then
+		title="";
+		coords="";
+	fi
+	temp=`echo "$line" | sed -n 's/\s*"title": "*\([^"]\+\).*/\1/p'`
+	if [[ "$temp" != "" ]]; then
+		title="$temp";
+	fi
+	temp=`echo "$line" | sed -n 's#.*<span class=.searchmatch.>Co<\/span>|\([0-9\-]\+\)|\([0-9\-]\+\)|\([0-9\-]\+\).*#\1,\3#p'`
+	if [[ "$temp" != "" ]]; then
+		coords="$temp";
+	fi
+	if [[ "$title" != "" && "$coords" != "" ]]; then
+		echo "{\"type\": \"Feature\", \"properties\": {\"name\": \"$title\","
+		echo -n "\"categories\": ["
+		urltitle=`echo "$title" | sed 's/ /%20/g'`
+		cjson=`curl -s "https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&titles=$urltitle&prop=categories" | json_reformat -m`
+		categories=`echo "$cjson" | sed -n 's/.*\("categories":[^]]\+\).*/\1/p'`;
+		IFS='}'
+		fc="true";
+		for entry in $categories; do
+			if [[ "$fc" != "true" ]]; then
+				echo -n ",";
+			fi
+			category=`echo "$entry" | sed -n 's/.*Category:\([^"]\+\).*/\1/p'`
+			echo -n "\"$category\"";
+			fc="false";
+		done;
+		IFS=$'\n'
+		echo "]},";
+		echo "\"geometry\": {\"type\": \"Point\", \"coordinates\": [$coords]}},";
+		title="";
+		coords="";
+	fi
+done
+IFS=" ";
+
+echo "{}]";
diff --git a/scripts/geojson/fetch_cities.sh b/scripts/geojson/fetch_cities.sh
deleted file mode 100755
index 16e81c1..0000000
--- a/scripts/geojson/fetch_cities.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-json=`curl 'https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:City&cmlimit=100'`
-cities=`echo "$json" | json_reformat | sed -n 's/.*"title":\s*"\([^"]*\).*/\1/p' | tr ' ' '_' | tr '\n' ' '`
-
-echo "["
-for city in $cities; do
-	./fetch_single.sh "$city"
-done
-echo "{}]"
diff --git a/scripts/geojson/fetch_shops.sh b/scripts/geojson/fetch_shops.sh
deleted file mode 100755
index 84a9f3f..0000000
--- a/scripts/geojson/fetch_shops.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-json=`curl 'https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:Shops&cmlimit=1000' 2>/dev/null`
-cities=`echo "$json" | json_reformat | sed -n 's/.*"title":\s*"\([^"]*\).*/\1/p' | tr ' ' '_' | tr '\n' ' '`
-
-echo "["
-for city in $cities; do
-	./fetch_single.sh "$city"
-done
-echo "{}]"
diff --git a/scripts/geojson/fetch_single.sh b/scripts/geojson/fetch_single.sh
deleted file mode 100755
index ce37b59..0000000
--- a/scripts/geojson/fetch_single.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-json=`curl "https://wiki.linux-forks.de/mediawiki/api.php?action=query&prop=revisions&rvprop=content&format=json&titles=$1&rvsection=0&rvslots=main" 2>/dev/null | sed -s "s/\\\\\\\\n//g"`
-
-title=`echo $json | sed -n 's/.*"title":\s*"\([^"]\+\).*/\1/p'`
-
-img=$(curl "https://wiki.linux-forks.de/mediawiki/index.php/$1" 2>/dev/null | sed s/\"/\\n/g | grep /thumb/ | head -n 1)
-
-image="https://wiki.linux-forks.de$img"
-
-coords=`echo "$json" | sed -n "s/.*coordinates = {{Co|\([^}]*\).*/\1/p"`
-coord_x=`echo "$coords" | sed -n "s/\([^|]\+\).*/\1/p"`
-coord_y=`echo "$coords" | sed -n "s/.*|\([^|]\+\).*/\1/p"`
-
-description=`curl "https://wiki.linux-forks.de/mediawiki/api.php?action=parse&page=$1&section=0&prop=text&format=json" 2>/dev/null | sed -z -n "s/<\/p><p>//g;s/.*<p>\(.*\)<\/p>.*/\1/p" | sed s,\"/mediawiki/index.php,\"https://wiki.linux-forks.de/mediawiki/index.php,g`
-
-
-if [[ "$coord_x" != "" && "$coord_y" != "" ]]; then
-	echo "{\
-		\"type\": \"Feature\",\
-		\"properties\": {\
-		\"name\": \"$title\",\
-		\"amenity\": \"City\",\
-		\"description\": \"$description\",\
-		\"image\": \"$image\"\
-		},\
-		\"geometry\": {\
-		\"type\": \"Point\",\
-		\"coordinates\": [$coord_x, $coord_y]\
-		}\
-	},"
-fi
diff --git a/scripts/geojson/fetch_stations.sh b/scripts/geojson/fetch_stations.sh
deleted file mode 100755
index 8b76428..0000000
--- a/scripts/geojson/fetch_stations.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-json=`curl 'https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:Stations&cmlimit=1000' 2>/dev/null`
-cities=`echo "$json" | json_reformat | sed -n 's/.*"title":\s*"\([^"]*\).*/\1/p' | tr ' ' '_' | tr '\n' ' '`
-
-echo "["
-for city in $cities; do
-	./fetch_single.sh "$city"
-done
-echo "{}]"
-- 
cgit v1.2.3