From af1090e3ca7cd9f38050b1c1c290bca56a9440dc Mon Sep 17 00:00:00 2001 From: theofficialvedantjoshi <vedant.messi101@gmail.com> Date: Wed, 8 Jan 2025 23:52:57 +0530 Subject: [PATCH 1/6] Feat: Update google oauth for google calendar integration. --- pyproject.toml | 3 + .../google/get-google-credentials | 68 ++++++++++++++----- zulip/integrations/google/google-calendar | 27 +++----- zulip/integrations/google/requirements.txt | 5 +- 4 files changed, 69 insertions(+), 34 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 159c85e23..5261b00f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,9 @@ module = [ "feedparser.*", "gitlint.*", "googleapiclient.*", + "google_api_python_client.*", + "google_auth_httplib2.*", + "google_auth_oauthlib.*", "irc.*", "mercurial.*", "nio.*", diff --git a/zulip/integrations/google/get-google-credentials b/zulip/integrations/google/get-google-credentials index bb97e5f69..00d693fb4 100755 --- a/zulip/integrations/google/get-google-credentials +++ b/zulip/integrations/google/get-google-credentials @@ -2,16 +2,35 @@ import argparse import os -from oauth2client import client, tools -from oauth2client.file import Storage +from google.auth.transport.requests import Request # type: ignore[import-not-found] +from google.oauth2.credentials import Credentials # type: ignore[import-not-found] +from google_auth_oauthlib.flow import InstalledAppFlow + +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument( + "--auth_host_name", default="localhost", help="Hostname when running a local web server." +) +parser.add_argument( + "--noauth_local_webserver", + action="store_true", + default=False, + help="Do not run a local web server.", +) +parser.add_argument( + "--auth_host_port", + default=[8080, 8090], + type=int, + nargs="*", + help="Port web server should listen on.", +) +flags = parser.parse_args() -flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args() # If modifying these scopes, delete your previously saved credentials # at zulip/bots/gcal/ # NOTE: When adding more scopes, add them after the previous one in the same field, with a space # seperating them. -SCOPES = "https://www.googleapis.com/auth/calendar.readonly" +SCOPES = ["https://www.googleapis.com/auth/calendar.readonly"] # This file contains the information that google uses to figure out which application is requesting # this client's data. CLIENT_SECRET_FILE = "client_secret.json" # noqa: S105 @@ -19,7 +38,7 @@ APPLICATION_NAME = "Zulip Calendar Bot" HOME_DIR = os.path.expanduser("~") -def get_credentials() -> client.Credentials: +def get_credentials() -> Credentials: """Gets valid user credentials from storage. If nothing has been stored, or if the stored credentials are invalid, @@ -28,19 +47,36 @@ def get_credentials() -> client.Credentials: Returns: Credentials, the obtained credential. """ - + credentials = None credential_path = os.path.join(HOME_DIR, "google-credentials.json") - - store = Storage(credential_path) - credentials = store.get() - if not credentials or credentials.invalid: - flow = client.flow_from_clientsecrets(os.path.join(HOME_DIR, CLIENT_SECRET_FILE), SCOPES) - flow.user_agent = APPLICATION_NAME - # This attempts to open an authorization page in the default web browser, and asks the user - # to grant the bot access to their data. If the user grants permission, the run_flow() - # function returns new credentials. - credentials = tools.run_flow(flow, store, flags) + if os.path.exists(credential_path): + credentials = Credentials.from_authorized_user_file(credential_path, SCOPES) + if not credentials or not credentials.valid: + if credentials and credentials.expired and credentials.refresh_token: + credentials.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + os.path.join(HOME_DIR, CLIENT_SECRET_FILE), SCOPES + ) + if not flags.noauth_local_webserver: + credentials = flow.run_local_server( + host=flags.auth_host_name, port=flags.auth_host_port[0] + ) + # This attempts to open an authorization page in the default web browser, and asks the user + # to grant the bot access to their data. If the user grants permission, the run_flow() + # function returns new credentials. + else: + auth_url, _ = flow.authorization_url(prompt="consent") + print( + "Proceed to the following link in your browser:", + auth_url, + ) + auth_code = input("Enter the authorization code: ") + credentials = flow.fetch_token(code=auth_code) + with open(credential_path, "w") as token: + token.write(credentials.to_json()) print("Storing credentials to " + credential_path) + return credentials get_credentials() diff --git a/zulip/integrations/google/google-calendar b/zulip/integrations/google/google-calendar index 85906bd46..032a4d9dc 100755 --- a/zulip/integrations/google/google-calendar +++ b/zulip/integrations/google/google-calendar @@ -12,21 +12,18 @@ import time from typing import List, Optional, Set, Tuple import dateutil.parser -import httplib2 import pytz -from oauth2client import client -from oauth2client.file import Storage try: - from googleapiclient import discovery + from google.oauth2.credentials import Credentials # type: ignore[import-not-found] + from googleapiclient.discovery import build except ImportError: - logging.exception("Install google-api-python-client") + logging.exception("Install google-api-python-client and google-auth-oauthlib") sys.exit(1) - sys.path.append(os.path.join(os.path.dirname(__file__), "../../")) import zulip -SCOPES = "https://www.googleapis.com/auth/calendar.readonly" +SCOPES = ["https://www.googleapis.com/auth/calendar.readonly"] CLIENT_SECRET_FILE = "client_secret.json" # noqa: S105 APPLICATION_NAME = "Zulip" HOME_DIR = os.path.expanduser("~") @@ -88,7 +85,7 @@ if not options.zulip_email: zulip_client = zulip.init_from_options(options) -def get_credentials() -> client.Credentials: +def get_credentials() -> Credentials: """Gets valid user credentials from storage. If nothing has been stored, or if the stored credentials are invalid, @@ -100,22 +97,20 @@ def get_credentials() -> client.Credentials: """ try: credential_path = os.path.join(HOME_DIR, "google-credentials.json") - - store = Storage(credential_path) - return store.get() - except client.Error: + credentials = Credentials.from_authorized_user_file(credential_path, SCOPES) + except ValueError: logging.exception("Error while trying to open the `google-credentials.json` file.") sys.exit(1) except OSError: logging.error("Run the get-google-credentials script from this directory first.") sys.exit(1) + else: + return credentials def populate_events() -> Optional[None]: - credentials = get_credentials() - creds = credentials.authorize(httplib2.Http()) - service = discovery.build("calendar", "v3", http=creds) - + creds = get_credentials() + service = build("calendar", "v3", credentials=creds) now = datetime.datetime.now(pytz.utc).isoformat() feed = ( service.events() diff --git a/zulip/integrations/google/requirements.txt b/zulip/integrations/google/requirements.txt index 139c0705b..018523c01 100644 --- a/zulip/integrations/google/requirements.txt +++ b/zulip/integrations/google/requirements.txt @@ -1,2 +1,3 @@ -httplib2>=0.22.0 -oauth2client>=4.1.3 +google-api-python-client>=2.157.0 +google-auth-httplib2>=0.2.0 +google-auth-oauthlib>=1.2.1 From 5c0b39482faeb8f69e8483297fbff40b440b276b Mon Sep 17 00:00:00 2001 From: theofficialvedantjoshi <vedant.messi101@gmail.com> Date: Thu, 6 Feb 2025 17:45:27 +0530 Subject: [PATCH 2/6] Fix: Remove unnecessary "sender" in request data. --- zulip/integrations/google/google-calendar | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zulip/integrations/google/google-calendar b/zulip/integrations/google/google-calendar index 032a4d9dc..ea3dffbd8 100755 --- a/zulip/integrations/google/google-calendar +++ b/zulip/integrations/google/google-calendar @@ -181,9 +181,7 @@ def send_reminders() -> Optional[None]: else: message = "Reminder:\n\n" + "\n".join("* " + m for m in messages) - zulip_client.send_message( - dict(type="private", to=options.zulip_email, sender=options.zulip_email, content=message) - ) + zulip_client.send_message(dict(type="private", to=options.zulip_email, content=message)) sent.update(keys) From 50a9579a1f4a9c2b5cd75207ddbe169660d26162 Mon Sep 17 00:00:00 2001 From: theofficialvedantjoshi <vedant.messi101@gmail.com> Date: Thu, 6 Feb 2025 23:01:24 +0530 Subject: [PATCH 3/6] Feat: Add more details to the reminder messages generated. --- zulip/integrations/google/google-calendar | 100 ++++++++++++++++++---- 1 file changed, 81 insertions(+), 19 deletions(-) diff --git a/zulip/integrations/google/google-calendar b/zulip/integrations/google/google-calendar index ea3dffbd8..4269a3c6a 100755 --- a/zulip/integrations/google/google-calendar +++ b/zulip/integrations/google/google-calendar @@ -9,7 +9,7 @@ import logging import os import sys import time -from typing import List, Optional, Set, Tuple +from typing import List, Optional, Set, Tuple, TypedDict import dateutil.parser import pytz @@ -28,8 +28,22 @@ CLIENT_SECRET_FILE = "client_secret.json" # noqa: S105 APPLICATION_NAME = "Zulip" HOME_DIR = os.path.expanduser("~") + +class Event(TypedDict): + id: int + start: datetime.datetime + end: datetime.datetime + summary: str + html_link: str + status: str + location: str + description: str + organizer: str + hangout_link: str + + # Our cached view of the calendar, updated periodically. -events: List[Tuple[int, datetime.datetime, str]] = [] +events: List[Event] = [] # Unique keys for events we've already sent, so we don't remind twice. sent: Set[Tuple[int, datetime.datetime]] = set() @@ -123,34 +137,84 @@ def populate_events() -> Optional[None]: ) .execute() ) - events.clear() for event in feed["items"]: try: start = dateutil.parser.parse(event["start"]["dateTime"]) + end = dateutil.parser.parse(event["end"]["dateTime"]) # According to the API documentation, a time zone offset is required # for start.dateTime unless a time zone is explicitly specified in # start.timeZone. - if start.tzinfo is None: + if start.tzinfo is None or end.tzinfo is None: event_timezone = pytz.timezone(event["start"]["timeZone"]) # pytz timezones include an extra localize method that's not part # of the tzinfo base class. start = event_timezone.localize(start) + end = event_timezone.localize(end) except KeyError: # All-day events can have only a date. start_naive = dateutil.parser.parse(event["start"]["date"]) - + end_naive = dateutil.parser.parse(event["end"]["date"]) # All-day events don't have a time zone offset; instead, we use the # time zone of the calendar. calendar_timezone = pytz.timezone(feed["timeZone"]) # pytz timezones include an extra localize method that's not part # of the tzinfo base class. start = calendar_timezone.localize(start_naive) + end = calendar_timezone.localize(end_naive) + id = event["id"] + summary = event.get("summary", "(No Title)") + html_link = event["htmlLink"] + status = event.get("status", "confirmed") + location = event.get("location", "") + description = event.get("description", "") + organizer = ( + "" + if ( + event["organizer"]["email"] == options.zulip_email or event["organizer"].get("self") + ) + else event["organizer"].get("displayName", event["organizer"]["email"]) + ) + hangout_link = event.get("hangoutLink", "") + events.append( + { + "id": id, + "start": start, + "end": end, + "summary": summary, + "html_link": html_link, + "status": status, + "location": location, + "description": description, + "organizer": organizer, + "hangout_link": hangout_link, + } + ) - try: - events.append((event["id"], start, event["summary"])) - except KeyError: - events.append((event["id"], start, "(No Title)")) + +def event_to_message(event: Event) -> str: + """Parse the event dictionary and return a string that can be sent as a message. + + The message includes the event title, start and end times, location, organizer, hangout link, and description. + + Returns: + str: The message to be sent. + """ + line = f"**[{event['summary']}]({event['html_link']})**\n" + if event["start"].hour == 0 and event["start"].minute == 0: + line += "Scheduled for today.\n" + else: + line += f"Scheduled from **{event['start'].strftime('%H:%M')}** to **{event['end'].strftime('%H:%M')}**.\n" + line += f"**Location:** {event['location']}\n" if event["location"] else "" + line += f"**Organizer:** {event['organizer']}\n" if event["organizer"] else "" + line += ( + f"**Hangout Link:** [{event['hangout_link'].split('/')[2]}]({event['hangout_link']})\n" + if event["hangout_link"] + else "" + ) + line += f"**Status:** {event['status']}\n" if event["status"] else "" + line += f"**Description:** {event['description']}\n" if event["description"] else "" + return line def send_reminders() -> Optional[None]: @@ -158,28 +222,26 @@ def send_reminders() -> Optional[None]: keys = set() now = datetime.datetime.now(tz=pytz.utc) - for id, start, summary in events: - dt = start - now + for event in events: + dt = event["start"] - now if dt.days == 0 and dt.seconds < 60 * options.interval: # The unique key includes the start time, because of # repeating events. - key = (id, start) + key = (event["id"], event["start"]) if key not in sent: - if start.hour == 0 and start.minute == 0: - line = f"{summary} is today." - else: - line = "{} starts at {}".format(summary, start.strftime("%H:%M")) + line = event_to_message(event) print("Sending reminder:", line) messages.append(line) keys.add(key) - if not messages: return if len(messages) == 1: - message = "Reminder: " + messages[0] + message = "**Reminder:**\n\n " + messages[0] else: - message = "Reminder:\n\n" + "\n".join("* " + m for m in messages) + message = "**Reminders:**\n\n" + "\n".join( + str(i + 1) + ". " + m for i, m in enumerate(messages) + ) zulip_client.send_message(dict(type="private", to=options.zulip_email, content=message)) From 12864e8f1ed2b47bce59319470fbf030a0acada0 Mon Sep 17 00:00:00 2001 From: theofficialvedantjoshi <vedant.messi101@gmail.com> Date: Fri, 7 Feb 2025 15:25:25 +0530 Subject: [PATCH 4/6] Feat: Use reminder overrides for events. --- zulip/integrations/google/google-calendar | 71 ++++++++++++++++------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/zulip/integrations/google/google-calendar b/zulip/integrations/google/google-calendar index 4269a3c6a..2a8662c82 100755 --- a/zulip/integrations/google/google-calendar +++ b/zulip/integrations/google/google-calendar @@ -40,13 +40,14 @@ class Event(TypedDict): description: str organizer: str hangout_link: str + reminder: int # Our cached view of the calendar, updated periodically. events: List[Event] = [] -# Unique keys for events we've already sent, so we don't remind twice. -sent: Set[Tuple[int, datetime.datetime]] = set() +# Unique keys for reminders we've already sent, so we don't remind twice. +sent: Set[Tuple[int, datetime.datetime, int]] = set() sys.path.append(os.path.dirname(__file__)) @@ -73,12 +74,11 @@ google-calendar --calendar calendarID@example.calendar.google.com parser.add_argument( - "--interval", - dest="interval", - default=30, + "--override", + dest="override", type=int, action="store", - help="Minutes before event for reminder [default: 30]", + help="Override the reminder time for all events.", metavar="MINUTES", ) @@ -125,13 +125,12 @@ def get_credentials() -> Credentials: def populate_events() -> Optional[None]: creds = get_credentials() service = build("calendar", "v3", credentials=creds) - now = datetime.datetime.now(pytz.utc).isoformat() feed = ( service.events() .list( calendarId=options.calendarID, - timeMin=now, - maxResults=5, + timeMin=datetime.datetime.now(pytz.utc).isoformat(), + timeMax=datetime.datetime.now(pytz.utc).isoformat().split("T")[0] + "T23:59:59Z", singleEvents=True, orderBy="startTime", ) @@ -162,6 +161,9 @@ def populate_events() -> Optional[None]: # of the tzinfo base class. start = calendar_timezone.localize(start_naive) end = calendar_timezone.localize(end_naive) + now = datetime.datetime.now(tz=start.tzinfo) + if start < now: + continue id = event["id"] summary = event.get("summary", "(No Title)") html_link = event["htmlLink"] @@ -176,7 +178,25 @@ def populate_events() -> Optional[None]: else event["organizer"].get("displayName", event["organizer"]["email"]) ) hangout_link = event.get("hangoutLink", "") - events.append( + reminders = event["reminders"] + # If the user has specified an override, we use that for all events. + # If the event uses the calendar's default reminders, we use that. + # If the event has overrides on Google Calendar, we use that. + # If none of the above, we don't set a reminder. + if options.override: + reminder_minutes = [options.override] + elif reminders.get("useDefault"): + calendar_list = service.calendarList().get(calendarId=options.calendarID).execute() + reminder_minutes = ( + [reminder["minutes"] for reminder in calendar_list["defaultReminders"]] + if calendar_list.get("defaultReminders") + else [] + ) + elif reminders.get("overrides"): + reminder_minutes = [reminder["minutes"] for reminder in reminders["overrides"]] + else: + reminder_minutes = [] + events.extend( { "id": id, "start": start, @@ -188,7 +208,9 @@ def populate_events() -> Optional[None]: "description": description, "organizer": organizer, "hangout_link": hangout_link, + "reminder": reminder, } + for reminder in reminder_minutes ) @@ -218,21 +240,30 @@ def event_to_message(event: Event) -> str: def send_reminders() -> Optional[None]: - messages = [] + messages: List[str] = [] keys = set() - now = datetime.datetime.now(tz=pytz.utc) - - for event in events: - dt = event["start"] - now - if dt.days == 0 and dt.seconds < 60 * options.interval: - # The unique key includes the start time, because of - # repeating events. - key = (event["id"], event["start"]) + # Sort events by the time of the reminder. + events.sort( + key=lambda event: (event["start"] - datetime.timedelta(minutes=event["reminder"])), + reverse=True, + ) + # Iterate through the events and send reminders for those whose reminder time has come or passed and remove them from the list. + # The instant a reminder's time is greater than the current time, we stop sending reminders and break out of the loop. + while len(events): + event = events[-1] + now = datetime.datetime.now(tz=event["start"].tzinfo) + dt = event["start"] - datetime.timedelta(minutes=event["reminder"]) + if dt <= now: + key = (event["id"], event["start"], event["reminder"]) if key not in sent: line = event_to_message(event) print("Sending reminder:", line) - messages.append(line) + messages = [line, *messages] keys.add(key) + events.pop() + else: + break + if not messages: return From 6a04715fd87daf49ad4a91da3727adc5a112166e Mon Sep 17 00:00:00 2001 From: theofficialvedantjoshi <vedant.messi101@gmail.com> Date: Sun, 23 Feb 2025 01:57:34 +0530 Subject: [PATCH 5/6] Feat: Handle `google.auth.exceptions.RefreshError`. --- .../google/get-google-credentials | 11 +++++++- zulip/integrations/google/google-calendar | 27 ++++++++++++------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/zulip/integrations/google/get-google-credentials b/zulip/integrations/google/get-google-credentials index 00d693fb4..5c76e9a1b 100755 --- a/zulip/integrations/google/get-google-credentials +++ b/zulip/integrations/google/get-google-credentials @@ -1,7 +1,10 @@ #!/usr/bin/env python3 import argparse +import logging import os +import sys +from google.auth.exceptions import RefreshError # type: ignore[import-not-found] from google.auth.transport.requests import Request # type: ignore[import-not-found] from google.oauth2.credentials import Credentials # type: ignore[import-not-found] from google_auth_oauthlib.flow import InstalledAppFlow @@ -53,7 +56,13 @@ def get_credentials() -> Credentials: credentials = Credentials.from_authorized_user_file(credential_path, SCOPES) if not credentials or not credentials.valid: if credentials and credentials.expired and credentials.refresh_token: - credentials.refresh(Request()) + try: + credentials.refresh(Request()) + except RefreshError: + logging.error( + "The credentials have expired. Generate a new client_secret.json file." + ) + sys.exit(1) else: flow = InstalledAppFlow.from_client_secrets_file( os.path.join(HOME_DIR, CLIENT_SECRET_FILE), SCOPES diff --git a/zulip/integrations/google/google-calendar b/zulip/integrations/google/google-calendar index 2a8662c82..ed869c0a5 100755 --- a/zulip/integrations/google/google-calendar +++ b/zulip/integrations/google/google-calendar @@ -15,6 +15,7 @@ import dateutil.parser import pytz try: + from google.auth.exceptions import RefreshError # type: ignore[import-not-found] from google.oauth2.credentials import Credentials # type: ignore[import-not-found] from googleapiclient.discovery import build except ImportError: @@ -125,17 +126,23 @@ def get_credentials() -> Credentials: def populate_events() -> Optional[None]: creds = get_credentials() service = build("calendar", "v3", credentials=creds) - feed = ( - service.events() - .list( - calendarId=options.calendarID, - timeMin=datetime.datetime.now(pytz.utc).isoformat(), - timeMax=datetime.datetime.now(pytz.utc).isoformat().split("T")[0] + "T23:59:59Z", - singleEvents=True, - orderBy="startTime", + try: + feed = ( + service.events() + .list( + calendarId=options.calendarID, + timeMin=datetime.datetime.now(pytz.utc).isoformat(), + timeMax=datetime.datetime.now(pytz.utc).isoformat().split("T")[0] + "T23:59:59Z", + singleEvents=True, + orderBy="startTime", + ) + .execute() ) - .execute() - ) + except RefreshError: + logging.error( + "The credentials have expired. Generate a new client_secret.json file and run the get-google-credentials script." + ) + sys.exit(1) events.clear() for event in feed["items"]: try: From 723c1c8a8a33307988b206b61db3064f02c33f41 Mon Sep 17 00:00:00 2001 From: theofficialvedantjoshi <vedant.messi101@gmail.com> Date: Sun, 23 Feb 2025 02:11:59 +0530 Subject: [PATCH 6/6] Feat: Replace `start_time` in sent set with `reminder_time`. --- zulip/integrations/google/google-calendar | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zulip/integrations/google/google-calendar b/zulip/integrations/google/google-calendar index ed869c0a5..19294f0d1 100755 --- a/zulip/integrations/google/google-calendar +++ b/zulip/integrations/google/google-calendar @@ -41,14 +41,14 @@ class Event(TypedDict): description: str organizer: str hangout_link: str - reminder: int + reminder: int # Minutes before the event to send a reminder. # Our cached view of the calendar, updated periodically. events: List[Event] = [] # Unique keys for reminders we've already sent, so we don't remind twice. -sent: Set[Tuple[int, datetime.datetime, int]] = set() +sent: Set[Tuple[int, int]] = set() sys.path.append(os.path.dirname(__file__)) @@ -261,7 +261,7 @@ def send_reminders() -> Optional[None]: now = datetime.datetime.now(tz=event["start"].tzinfo) dt = event["start"] - datetime.timedelta(minutes=event["reminder"]) if dt <= now: - key = (event["id"], event["start"], event["reminder"]) + key = (event["id"], event["reminder"]) if key not in sent: line = event_to_message(event) print("Sending reminder:", line)