diff options
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/geojson/fetch.sh | 30 | ||||
-rwxr-xr-x | scripts/geojson/fetch_all_points.sh | 48 | ||||
-rwxr-xr-x | scripts/geojson/fetch_cities.sh | 10 | ||||
-rwxr-xr-x | scripts/geojson/fetch_shops.sh | 10 | ||||
-rwxr-xr-x | scripts/geojson/fetch_single.sh | 31 | ||||
-rwxr-xr-x | scripts/geojson/fetch_stations.sh | 10 |
6 files changed, 48 insertions, 91 deletions
diff --git a/scripts/geojson/fetch.sh b/scripts/geojson/fetch.sh deleted file mode 100755 index bd86239..0000000 --- a/scripts/geojson/fetch.sh +++ /dev/null @@ -1,30 +0,0 @@ -fetch_category () { - echo "Fetching $1" > /dev/stderr - json=`curl "https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:$1&cmlimit=max" 2>/dev/null` - cities=`echo "$json" | json_reformat | sed -n 's/.*"title":\s*"\([^"]*\).*/\1/p' | tr ' ' '_' | tr '\n' ' '` - num=`echo $cities | tr ' ' '\n' | wc -l` - i=0 - echo "[" - for city in $cities; do - let i++ - echo -n -e "\rFetched $i/$num" > /dev/stderr - ./fetch_single.sh "$city" - done - echo > /dev/stderr - echo "{}]" -} - -JSONDIR=../../htdocs/geojson - -mkdir -p $JSONDIR - - -fetch_category Shops > $JSONDIR/shops.json -fetch_category City > $JSONDIR/cities.json -fetch_category Stations > $JSONDIR/stations.json -fetch_category Parks > $JSONDIR/parks.json -fetch_category Libraries > $JSONDIR/libraries.json -fetch_category "CW_Complexes" > $JSONDIR/cw_complexes.json -fetch_category Courts > $JSONDIR/courts.json -fetch_category Waterway > $JSONDIR/waterway.json -fetch_category Train_Depots > $JSONDIR/depots.json diff --git a/scripts/geojson/fetch_all_points.sh b/scripts/geojson/fetch_all_points.sh new file mode 100755 index 0000000..81c46e0 --- /dev/null +++ b/scripts/geojson/fetch_all_points.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +json=`curl -s 'https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=search&redirects=1&converttitles=1&srsearch=%7B%7BCo%7C&srlimit=max&srwhat=text&srprop=snippet'` +data=`echo "$json" | json_reformat`; + +echo "["; + +IFS=$'\n'; +for line in $data; do + temp=`echo "$line" | sed -n 's/^\s*{\s*$/END/p'` + if [[ "$temp" == "END" ]]; then + title=""; + coords=""; + fi + temp=`echo "$line" | sed -n 's/\s*"title": "*\([^"]\+\).*/\1/p'` + if [[ "$temp" != "" ]]; then + title="$temp"; + fi + temp=`echo "$line" | sed -n 's#.*<span class=.searchmatch.>Co<\/span>|\([0-9\-]\+\)|\([0-9\-]\+\)|\([0-9\-]\+\).*#\1,\3#p'` + if [[ "$temp" != "" ]]; then + coords="$temp"; + fi + if [[ "$title" != "" && "$coords" != "" ]]; then + echo "{\"type\": \"Feature\", \"properties\": {\"name\": \"$title\"," + echo -n "\"categories\": [" + urltitle=`echo "$title" | sed 's/ /%20/g'` + cjson=`curl -s "https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&titles=$urltitle&prop=categories" | json_reformat -m` + categories=`echo "$cjson" | sed -n 's/.*\("categories":[^]]\+\).*/\1/p'`; + IFS='}' + fc="true"; + for entry in $categories; do + if [[ "$fc" != "true" ]]; then + echo -n ","; + fi + category=`echo "$entry" | sed -n 's/.*Category:\([^"]\+\).*/\1/p'` + echo -n "\"$category\""; + fc="false"; + done; + IFS=$'\n' + echo "]},"; + echo "\"geometry\": {\"type\": \"Point\", \"coordinates\": [$coords]}},"; + title=""; + coords=""; + fi +done +IFS=" "; + +echo "{}]"; diff --git a/scripts/geojson/fetch_cities.sh b/scripts/geojson/fetch_cities.sh deleted file mode 100755 index 16e81c1..0000000 --- a/scripts/geojson/fetch_cities.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -json=`curl 'https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:City&cmlimit=100'` -cities=`echo "$json" | json_reformat | sed -n 's/.*"title":\s*"\([^"]*\).*/\1/p' | tr ' ' '_' | tr '\n' ' '` - -echo "[" -for city in $cities; do - ./fetch_single.sh "$city" -done -echo "{}]" diff --git a/scripts/geojson/fetch_shops.sh b/scripts/geojson/fetch_shops.sh deleted file mode 100755 index 84a9f3f..0000000 --- a/scripts/geojson/fetch_shops.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -json=`curl 'https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:Shops&cmlimit=1000' 2>/dev/null` -cities=`echo "$json" | json_reformat | sed -n 's/.*"title":\s*"\([^"]*\).*/\1/p' | tr ' ' '_' | tr '\n' ' '` - -echo "[" -for city in $cities; do - ./fetch_single.sh "$city" -done -echo "{}]" diff --git a/scripts/geojson/fetch_single.sh b/scripts/geojson/fetch_single.sh deleted file mode 100755 index ce37b59..0000000 --- a/scripts/geojson/fetch_single.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -json=`curl "https://wiki.linux-forks.de/mediawiki/api.php?action=query&prop=revisions&rvprop=content&format=json&titles=$1&rvsection=0&rvslots=main" 2>/dev/null | sed -s "s/\\\\\\\\n//g"` - -title=`echo $json | sed -n 's/.*"title":\s*"\([^"]\+\).*/\1/p'` - -img=$(curl "https://wiki.linux-forks.de/mediawiki/index.php/$1" 2>/dev/null | sed s/\"/\\n/g | grep /thumb/ | head -n 1) - -image="https://wiki.linux-forks.de$img" - -coords=`echo "$json" | sed -n "s/.*coordinates = {{Co|\([^}]*\).*/\1/p"` -coord_x=`echo "$coords" | sed -n "s/\([^|]\+\).*/\1/p"` -coord_y=`echo "$coords" | sed -n "s/.*|\([^|]\+\).*/\1/p"` - -description=`curl "https://wiki.linux-forks.de/mediawiki/api.php?action=parse&page=$1§ion=0&prop=text&format=json" 2>/dev/null | sed -z -n "s/<\/p><p>//g;s/.*<p>\(.*\)<\/p>.*/\1/p" | sed s,\"/mediawiki/index.php,\"https://wiki.linux-forks.de/mediawiki/index.php,g` - - -if [[ "$coord_x" != "" && "$coord_y" != "" ]]; then - echo "{\ - \"type\": \"Feature\",\ - \"properties\": {\ - \"name\": \"$title\",\ - \"amenity\": \"City\",\ - \"description\": \"$description\",\ - \"image\": \"$image\"\ - },\ - \"geometry\": {\ - \"type\": \"Point\",\ - \"coordinates\": [$coord_x, $coord_y]\ - }\ - }," -fi diff --git a/scripts/geojson/fetch_stations.sh b/scripts/geojson/fetch_stations.sh deleted file mode 100755 index 8b76428..0000000 --- a/scripts/geojson/fetch_stations.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -json=`curl 'https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:Stations&cmlimit=1000' 2>/dev/null` -cities=`echo "$json" | json_reformat | sed -n 's/.*"title":\s*"\([^"]*\).*/\1/p' | tr ' ' '_' | tr '\n' ' '` - -echo "[" -for city in $cities; do - ./fetch_single.sh "$city" -done -echo "{}]" |