From 7355b3e3503ca95e1ee11ebb814c3f2551143a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20P=C3=A9rez-Cerezo?= Date: Thu, 1 Mar 2018 17:47:10 +0100 Subject: Bumped version to 0.3, added html formatter --- bin/mensa | 27 ++++++++++++---- mensa/backends/studentenwerk.py | 35 ++++++++++---------- mensa/frontends/html.py | 33 +++++++++++++++++++ mensa/frontends/html.yapsy-plugin | 3 ++ mensa/frontends/plain-text.py | 4 +++ mensa/logic.py | 68 ++++++++++++++++++++++++++++++--------- setup.py | 3 +- 7 files changed, 132 insertions(+), 41 deletions(-) create mode 100644 mensa/frontends/html.py create mode 100644 mensa/frontends/html.yapsy-plugin diff --git a/bin/mensa b/bin/mensa index 0fa4d81..0ccad0c 100755 --- a/bin/mensa +++ b/bin/mensa @@ -5,13 +5,16 @@ from mensa import logic from mensa import base parser = argparse.ArgumentParser(description='Fetch menus from various sources') -# parser.add_argument('integers', metavar='N', type=int, nargs='+', -# help='an integer for the accumulator') -parser.add_argument('-r', '--restaurants', dest='rest', action='store', - metavar='LIST', - help='Comma-separated list of restaurants to fetch the menus from.') +parser.add_argument('rest', nargs="*", + metavar='RESTAURANT', + help='Fetch menus from this restaurant') parser.add_argument('-l', '--list-restaurants', dest='list', action='store_true', help='get list of restaurants') +parser.add_argument('-f', '--formatter', dest='form', action='store', nargs=1, help="formatter to use") +parser.add_argument('-g', '--vegetarian', dest='vegetarian', action='store_true', + help='show only vegetarian meals') +parser.add_argument('-G', '--vegan', dest='vegan', action='store_true', + help='show only vegan meals') args = parser.parse_args() @@ -19,6 +22,16 @@ args = parser.parse_args() logic.init_foodsources() ## Load frontends (not yet implemented) logic.init_renderers() +veggie = 0 +form = ["plain-text"] +if args.form : + form = args.form + + +if args.vegetarian : + veggie = 1 +if args.vegan : + veggie = 2 @@ -28,7 +41,7 @@ if args.list : exit() restlist = None if args.rest : - restlist = args.rest.split(",") + restlist = args.rest foodl = logic.get_food(restlist) -logic.render(foodl, ["plain-text"]) +logic.render(foodl, form, only_veggie=veggie) diff --git a/mensa/backends/studentenwerk.py b/mensa/backends/studentenwerk.py index a8f6c2b..f916096 100644 --- a/mensa/backends/studentenwerk.py +++ b/mensa/backends/studentenwerk.py @@ -8,33 +8,28 @@ import html5lib from mensa.base import * from yapsy.IPlugin import IPlugin import multiprocessing - +import datetime from yapsy import NormalizePluginNameForModuleName as normalize -mensenliste = {"TU Hardenbergstraße" : "mensa-tu-hardenbergstra%C3%9Fe", "TU Marchstraße": "cafeteria-tu-marchstra%C3%9Fe", "TU Skyline": "cafeteria-tu-skyline", "TU Architektur": "cafeteria-tu-architektur", "TU Ackerstraße": "cafeteria-tu-ackerstra%C3%9Fe"} - - - -def pr_f(j) : - i,k = j - food = get_food_items(k, ignore_nudelauswahl=True) - return (i,"*"*20+i+"*"*20+"\n"+formt(food)) - class Studentenwerk(IPlugin) : - def register_restaurants (self) : - mensenliste = {"TU Hardenbergstraße" : "mensa-tu-hardenbergstra%C3%9Fe", "TU Marchstraße": "cafeteria-tu-marchstra%C3%9Fe", "TU Skyline": "cafeteria-tu-skyline", "TU Architektur": "cafeteria-tu-architektur", "TU Ackerstraße": "cafeteria-tu-ackerstra%C3%9Fe"} - for h,n in mensenliste.items() : - r = Restaurant(normalize(h), h, self, "dummy", [n]) - register_restaurant(r) - def get_food_items(self, mensa="mensa-tu-hardenbergstra%C3%9Fe", ignore_nudelauswahl=False) : + def fetch_page(self, mensa) : user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)' headers = {'User-Agent': user_agent} req = urllib.request.Request('https://www.stw.berlin/mensen/%s.html' % mensa, headers=headers) response = urllib.request.urlopen(req) the_page = response.read() + open("/tmp/the_page","w").write(str(the_page)) document = html5lib.parse(the_page, treebuilder="lxml") + return document + def register_restaurants (self) : + mensenliste = {"TU Hardenbergstraße" : "mensa-tu-hardenbergstra%C3%9Fe", "TU Marchstraße": "cafeteria-tu-marchstra%C3%9Fe", "TU Skyline": "cafeteria-tu-skyline", "TU Architektur": "cafeteria-tu-architektur", "TU Ackerstraße": "cafeteria-tu-ackerstra%C3%9Fe"} + for h,n in mensenliste.items() : + r = Restaurant(normalize(h), h, self, "dummy", [n]) + register_restaurant(r) + def get_food_items(self, mensa="mensa-tu-hardenbergstra%C3%9Fe", ignore_nudelauswahl=False) : + document = self.fetch_page(mensa) groupsel = CSSSelector('.splGroupWrapper') groups = [e for e in groupsel(document)] fl = [] @@ -60,7 +55,13 @@ class Studentenwerk(IPlugin) : price = pricesel(m)[-1].text.strip() fl.append(Food(nm, price, name, veg)) return fl - + def get_opening_hours(self, mensa) : + #### Rudiment of a function for getting opening hours. Does NOT work yet due to unknown issues. + doc = self.fetch_page(mensa) + groupsel = CSSSelector('div.col-xs-10') + groups = [e for e in groupsel(doc)] + print(groups) + return doc # format: diff --git a/mensa/frontends/html.py b/mensa/frontends/html.py new file mode 100644 index 0000000..8ffec37 --- /dev/null +++ b/mensa/frontends/html.py @@ -0,0 +1,33 @@ +from mensa import base +from yapsy.IPlugin import IPlugin +from xml.sax.saxutils import escape as esc + +class HTMLRenderer(IPlugin) : + def render (self, foods, **options) : + ## Expects list of tuples with (Restaurant, Foodlist) + r = "" + vegkeys = [ "", "Vegetarian", "Vegan" ] + for restaurant, food in foods : + cat = [] + if not food : + continue + r = r+"

"+esc(restaurant.human_name)+"

"+"\n"#+base.formt(food) + food.sort(key=lambda foo: foo.category) + for i in food: + if options["only_veggie"] and options["only_veggie"] > i.veggie : + continue + if not i.category in cat : + if cat : + r = r + "" + cat.append(i.category) + if not i.category == None : + r=r+ "

"+esc(i.category)+"

" + r = r + print(r) + + def register_renderer(self) : + base.register_renderer(base.Renderer("html", "HTML Renderer", self)) diff --git a/mensa/frontends/html.yapsy-plugin b/mensa/frontends/html.yapsy-plugin new file mode 100644 index 0000000..135ad17 --- /dev/null +++ b/mensa/frontends/html.yapsy-plugin @@ -0,0 +1,3 @@ +[Core] +Name = HTML Output +Module = html \ No newline at end of file diff --git a/mensa/frontends/plain-text.py b/mensa/frontends/plain-text.py index 501be54..7a31371 100644 --- a/mensa/frontends/plain-text.py +++ b/mensa/frontends/plain-text.py @@ -7,9 +7,13 @@ class TextRenderer(IPlugin) : vegkeys = [ "", "Vegetarian", "Vegan" ] for restaurant, food in foods : cat = [] + if not food : + continue r = r+"*"*20+restaurant.human_name+"*"*20+"\n"#+base.formt(food) food.sort(key=lambda foo: foo.category) for i in food: + if options["only_veggie"] and options["only_veggie"] > i.veggie : + continue if not i.category in cat : cat.append(i.category) if not i.category == None : diff --git a/mensa/logic.py b/mensa/logic.py index f14b89c..75b2465 100755 --- a/mensa/logic.py +++ b/mensa/logic.py @@ -3,7 +3,13 @@ from mensa import base import urllib.error import os +import sys from yapsy.PluginManager import PluginManager +try: + import multiprocessing + parallel=True +except: + parallel = False def init_foodsources(): backends = PluginManager() @@ -22,25 +28,55 @@ def init_renderers(): pluginInfo.plugin_object.register_renderer() -def get_food(restlist=False, options={}) : +def get_food(restlist=False, **options) : foodl = [] - for k,i in base.foodsources.items() : - if restlist and not i.name in restlist : - continue - try : - food = i.get_food(ignore_nudelauswahl=True) - foodl.append((i, food)) - # print("*"*20+i.human_name+"*"*20+"\n"+base.formt(food)) - except base.NoMenuError: - print(i.human_name + ": No menu found. This could be due to a holiday or due to an error in the script.") - except urllib.error.HTTPError as e : - print(i.human_name + ": Fetching menu failed: %s" % str(e)) - except : - print(i.human_name + ": Unknown error") + if parallel : + r = [] + for k,i in base.foodsources.items() : + if restlist and not i.name in restlist : + continue + r.append(i) + + foodl = get_food_parallel(r, ignore_nudelauswahl=True) + else : + for k,i in base.foodsources.items() : + if restlist and not i.name in restlist : + continue + try : + food = i.get_food(ignore_nudelauswahl=True) + foodl.append((i, food)) + # print("*"*20+i.human_name+"*"*20+"\n"+base.formt(food)) + except base.NoMenuError: + sys.stderr.write(i.human_name + ": No menu found. This could be due to a holiday or due to an error in the script.\n") + except urllib.error.HTTPError as e : + sys.stderr.write(i.human_name + ": Fetching menu failed: %s\n" % str(e)) + except : + sys.stderr.write(i.human_name + ": Unknown error\n") return foodl -def render(to_render, rendlist=False, options={}) : +# format: +# if __name__ == "__main__": +# pool = multiprocessing.Pool(4) +# k = pool.map(pr_f, list(mensenliste.items())) +# k.sort(key=lambda x: x[0]) +# for i in k : +# print(i[1]) +def get_food_parallel_helper(i) : + try : + return (i[0],i[0].get_food(**i[1])) + except: + sys.stderr.write(i[0].human_name + ": No menu found. This could be due to a holiday or due to an error in the script.\n") + return (i[0], []) + +def get_food_parallel(rl, **options) : + pool = multiprocessing.Pool(60) + k = [ (i, options) for i in rl ] + l = list(pool.map(get_food_parallel_helper, k)) + l.sort(key=lambda x: x[0].human_name) + return l + +def render(to_render, rendlist=False, **options) : for k,i in base.renderers.items() : if rendlist and not i.name in rendlist : continue - i.render(to_render) + i.render(to_render, **options) diff --git a/setup.py b/setup.py index c28954b..2ec44af 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ def read(fname): setup( name = "mensa", - version = "0.1.3", + version = "0.3", author = "Gabriel Pérez-Cerezo", author_email = "gabriel@gpcf.eu", description = ("A program that fetches menus from various restaurants. Pre-installed by default are various cafeterias around TU Berlin."), @@ -21,6 +21,7 @@ setup( scripts=['bin/mensa'], long_description=read('README'), include_package_data=True, + install_requires=['yapsy','lxml', 'cssselect'], classifiers=[ "Development Status :: 3 - Alpha", "Topic :: Utilities", -- cgit v1.2.3