Skip to content
Snippets Groups Projects
Commit f8368d3f authored by fmk17's avatar fmk17
Browse files

Fix crawler, logs and upgrade requirements

parent 61ee46eb
No related branches found
No related tags found
No related merge requests found
......@@ -17,7 +17,7 @@ LocationDays = namedtuple("LocationDays", "days, location, update_datetime")
cached_update_times: Dict[Location, datetime] = dict()
cached_responses: Dict[Location, LocationDays] = dict()
DATE_REGEX = r": (\d{1,2})\/(\d{1,2})\/(\d{2,4})"
DATE_REGEX = r"(\d{1,2})\/(\d{1,2})\/(\d{2,4})"
logger = logging.getLogger("crawler")
......@@ -38,7 +38,11 @@ def get_location_days(
soup = BeautifulSoup(response.text, "lxml")
post = soup.select_one("#post div:nth-child(3)")
post_children = iter(post.children)
grouper = post.select_one('.wp-block-group__inner-container')
if grouper is not None:
post_children = iter(grouper.children)
else:
post_children = iter(post.children)
post_children = (node for node in post_children if node.text.strip() != "")
post_children = dropwhile(
lambda n: not re.search(DATE_REGEX, n.text), post_children
......@@ -85,7 +89,10 @@ def get_location_days(
if indicator is not None:
item_indicators.add(indicator)
elif child.text.strip() != "":
item_name = child.text.strip()
if item_name is not None:
item_name += " " + child.text.strip()
else:
item_name = child.text.strip()
if item_name is not None:
items.append(
MenuItem(
......
......@@ -567,7 +567,7 @@ def answer_delete_one_notification(user, data):
else:
logger.info(
f"User {format_user(user)} removed"
f" location={data.location_name}, meal={data.meal_name},"
f" location={data.location.name}, meal={data.meal_name},"
f" week_day={data.week_day.value}, time={data.time}"
)
delete_schedule_by_key(
......@@ -745,7 +745,7 @@ def answer_schedule_before_opening_callback(
)
logger.info(
f"User {format_user(update.effective_user)}"
" added schedule {pformat(schedule)}"
f" added schedule {pformat(schedule)}"
)
upsert_schedule(schedule)
body, keyboard = answer_schedule(update.effective_user.id)
......@@ -985,7 +985,7 @@ def main() -> None:
updater.dispatcher.job_queue.run_repeating(
send_scheduled, timedelta(minutes=EVERY_X_MINUTES), first=time
)
updater.start_polling()
updater.start_polling(timeout=600)
logger.info("Connected")
updater.idle()
logger.info("Done")
......
......@@ -6,7 +6,7 @@ certifi==2021.10.8
charset-normalizer==2.0.11
idna==3.3
python-dotenv==0.19.2
python-telegram-bot==13.11
python-telegram-bot==13.14
pytz==2021.3
pytz-deprecation-shim==0.1.0.post0
requests==2.27.1
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment