Rework member chunking and events (#662)

Author: dolfies

discord/abc.py

@@ -622,6 +622,21 @@ def __str__(self) -> str:
     def _sorting_bucket(self) -> int:
         raise NotImplementedError
 
+    @property
+    def member_list_id(self) -> Union[str, Literal["everyone"]]:
+        if self.permissions_for(self.guild.default_role).read_messages:
+            return "everyone"
+
+        overwrites = []
+        for overwrite in self._overwrites:
+            allow, deny = Permissions(overwrite.allow), Permissions(overwrite.deny)
+            if allow.read_messages:
+                overwrites.append(f"allow:{overwrite.id}")
+            elif deny.read_messages:
+                overwrites.append(f"deny:{overwrite.id}")
+
+        return str(utils.murmurhash32(",".join(sorted(overwrites)), signed=False))
+
     def _update(self, guild: Guild, data: Dict[str, Any]) -> None:
         raise NotImplementedError
 

discord/channel.py

@@ -3458,7 +3458,6 @@ class DMChannel(discord.abc.Messageable, discord.abc.Connectable, discord.abc.Pr
         '_requested_at',
         '_spam',
         '_state',
-        '_accessed',
     )
 
     def __init__(self, *, me: ClientUser, state: ConnectionState, data: DMChannelPayload):
@@ -3467,7 +3466,6 @@ def __init__(self, *, me: ClientUser, state: ConnectionState, data: DMChannelPay
         self.me: ClientUser = me
         self.id: int = int(data['id'])
         self._update(data)
-        self._accessed: bool = False
 
     def _update(self, data: DMChannelPayload) -> None:
         self.last_message_id: Optional[int] = utils._get_as_snowflake(data, 'last_message_id')
@@ -3486,9 +3484,6 @@ def _add_call(self, **kwargs) -> PrivateCall:
         return PrivateCall(**kwargs)
 
     async def _get_channel(self) -> Self:
-        if not self._accessed:
-            await self._state.call_connect(self.id)
-            self._accessed = True
         return self
 
     async def _initial_ring(self) -> None:
@@ -3912,15 +3907,13 @@ class GroupChannel(discord.abc.Messageable, discord.abc.Connectable, discord.abc
         'name',
         'me',
         '_state',
-        '_accessed',
     )
 
     def __init__(self, *, me: ClientUser, state: ConnectionState, data: GroupChannelPayload):
         self._state: ConnectionState = state
         self.id: int = int(data['id'])
         self.me: ClientUser = me
         self._update(data)
-        self._accessed: bool = False
 
     def _update(self, data: GroupChannelPayload) -> None:
         self.owner_id: int = int(data['owner_id'])
@@ -3940,9 +3933,6 @@ def _get_voice_state_pair(self) -> Tuple[int, int]:
         return self.me.id, self.id
 
     async def _get_channel(self) -> Self:
-        if not self._accessed:
-            await self._state.call_connect(self.id)
-            self._accessed = True
         return self
 
     def _initial_ring(self):

discord/client.py

@@ -178,10 +178,38 @@ class Client:
         amounts of guilds. The default is ``True``.
 
         .. versionadded:: 1.5
+    guild_subscriptions: :class:`bool`
+        Whether to subscribe to all guilds at startup.
+        This is required to receive member events and populate the thread cache.
+
+        For larger servers, this is required to receive nearly all events.
+
+        See :doc:`guild_subscriptions` for more information.
+
+        .. versionadded:: 2.1
+
+        .. warning::
+
+            If this is set to ``False``, the following consequences will occur:
+
+            - Large guilds (over 75,000 members) will not dispatch any non-stateful events (e.g. :func:`.on_message`, :func:`.on_reaction_add`, :func:`.on_typing`, etc.)
+            - :attr:`~Guild.threads` will only contain threads the client has joined.
+            - Guilds will not be chunkable and member events (e.g. :func:`.on_member_update`) will not be dispatched.
+                - Most :func:`.on_user_update` occurences will not be dispatched.
+                - The member (:attr:`~Guild.members`) and user (:attr:`~Client.users`) cache will be largely incomplete.
+                - Essentially, only the client user, friends/implicit relationships, voice members, and other subscribed-to users will be cached and dispatched.
+
+            This is useful if you want to control subscriptions manually (see :meth:`Guild.subscribe`) to save bandwidth and memory.
+            Disabling this is not recommended for most use cases.
     request_guilds: :class:`bool`
-        Whether to request guilds at startup. Defaults to True.
+        See ``guild_subscriptions``.
 
         .. versionadded:: 2.0
+
+        .. deprecated:: 2.1
+
+            This is deprecated and will be removed in a future version.
+            Use ``guild_subscriptions`` instead.
     status: Optional[:class:`.Status`]
         A status to start your presence with upon logging on to Discord.
     activity: Optional[:class:`.BaseActivity`]
@@ -972,7 +1000,7 @@ def clear(self) -> None:
         """
         self._closed = False
         self._ready.clear()
-        self._connection.clear()
+        self._connection.clear(full=True)
         self.http.clear()
 
     async def start(self, token: str, *, reconnect: bool = True) -> None:

discord/experiment.py

@@ -429,7 +429,9 @@ def __len__(self) -> int:
         return len(self._ids)
 
     def __contains__(self, item: Union[int, Snowflake], /) -> bool:
-        return getattr(item, 'id', item) in self._ids
+        if isinstance(item, int):
+            return item in self._ids
+        return item.id in self._ids
 
     def __iter__(self) -> Iterator[int]:
         return iter(self._ids)

discord/flags.py

@@ -301,7 +301,20 @@ class Capabilities(BaseFlags):
     @classmethod
     def default(cls: Type[Self]) -> Self:
         """Returns a :class:`Capabilities` with the current value used by the library."""
-        return cls._from_value(8189)
+        return cls(
+            lazy_user_notes=True,
+            versioned_read_states=True,
+            versioned_user_guild_settings=True,
+            dedupe_user_objects=True,
+            prioritized_ready_payload=True,
+            multiple_guild_experiment_populations=True,
+            non_channel_read_states=True,
+            auth_token_refresh=True,
+            user_settings_proto=True,
+            client_state_v2=True,
+            passive_guild_update=True,
+            auto_call_connect=True,
+        )
 
     @flag_value
     def lazy_user_notes(self):
@@ -365,10 +378,16 @@ def passive_guild_update(self):
         return 1 << 11
 
     @flag_value
-    def unknown_12(self):
-        """:class:`bool`: Unknown."""
+    def auto_call_connect(self):
+        """:class:`bool`: Connect user to all existing calls on connect (deprecates ``CALL_CONNECT`` opcode)."""
         return 1 << 12
 
+    @flag_value
+    def debounce_message_reactions(self):
+        """:class:`bool`: Debounce message reactions (dispatches ``MESSAGE_REACTION_ADD_MANY`` instead of ``MESSAGE_REACTION_ADD`` when a lot of reactions are sent in quick succession)."""
+        # Debounced reactions don't have member information, so this is kinda undesirable :(
+        return 1 << 13
+
 
 @fill_with_flags(inverted=True)
 class SystemChannelFlags(BaseFlags):
@@ -1171,23 +1190,17 @@ def voice(self):
         return 1
 
     @flag_value
-    def other(self):
-        """:class:`bool`: Whether to cache members that are collected from other means.
-
-        This does not apply to members explicitly cached (e.g. :attr:`Guild.chunk`, :attr:`Guild.fetch_members`).
+    def joined(self):
+        """:class:`bool`: Whether to cache members that joined the guild
+        or are chunked as part of the initial log in flow.
 
-        There is an alias for this called :attr:`joined`.
+        Members that leave the guild are no longer cached.
         """
         return 2
 
     @alias_flag_value
-    def joined(self):
-        """:class:`bool`: Whether to cache members that are collected from other means.
-
-        This does not apply to members explicitly cached (e.g. :attr:`Guild.chunk`, :attr:`Guild.fetch_members`).
-
-        This is an alias for :attr:`other`.
-        """
+    def other(self):
+        """:class:`bool`: Alias for :attr:`joined`."""
         return 2
 
     @property

discord/gateway.py

@@ -61,6 +61,7 @@
     from .enums import Status
     from .state import ConnectionState
     from .types.snowflake import Snowflake
+    from .types.gateway import BulkGuildSubscribePayload
     from .voice_client import VoiceClient
 
 
@@ -238,45 +239,10 @@ class DiscordWebSocket:
 
     Attributes
     -----------
-    DISPATCH
-        Receive only. Denotes an event to be sent to Discord, such as READY.
-    HEARTBEAT
-        When received tells Discord to keep the connection alive.
-        When sent asks if your connection is currently alive.
-    IDENTIFY
-        Send only. Starts a new session.
-    PRESENCE
-        Send only. Updates your presence.
-    VOICE_STATE
-        Send only. Starts a new connection to a voice guild.
-    VOICE_PING
-        Send only. Checks ping time to a voice guild, do not use.
-    RESUME
-        Send only. Resumes an existing connection.
-    RECONNECT
-        Receive only. Tells the client to reconnect to a new gateway.
-    REQUEST_MEMBERS
-        Send only. Asks for the guild members.
-    INVALIDATE_SESSION
-        Receive only. Tells the client to optionally invalidate the session
-        and IDENTIFY again.
-    HELLO
-        Receive only. Tells the client the heartbeat interval.
-    HEARTBEAT_ACK
-        Receive only. Confirms receiving of a heartbeat. Not having it implies
-        a connection issue.
-    GUILD_SYNC
-        Send only. Requests a guild sync. This is unfortunately no longer functional.
-    CALL_CONNECT
-        Send only. Maybe used for calling? Probably just tracking.
-    GUILD_SUBSCRIBE
-        Send only. Subscribes you to guilds/guild members. Might respond with GUILD_MEMBER_LIST_UPDATE.
-    REQUEST_COMMANDS
-        Send only. Requests application commands from a guild. Responds with GUILD_APPLICATION_COMMANDS_UPDATE.
     gateway
         The gateway we are currently connected to.
     token
-        The authentication token for discord.
+        The authentication token for Discord.
     """
 
     if TYPE_CHECKING:
@@ -307,11 +273,12 @@ class DiscordWebSocket:
     INVALIDATE_SESSION    = 9
     HELLO                 = 10
     HEARTBEAT_ACK         = 11
-    GUILD_SYNC            = 12  # :(
+    # GUILD_SYNC          = 12
     CALL_CONNECT          = 13
-    GUILD_SUBSCRIBE       = 14
-    REQUEST_COMMANDS      = 24
+    GUILD_SUBSCRIBE       = 14  # Deprecated
+    # REQUEST_COMMANDS    = 24
     SEARCH_RECENT_MEMBERS = 35
+    BULK_GUILD_SUBSCRIBE  = 37
     # fmt: on
 
     def __init__(self, socket: aiohttp.ClientWebSocketResponse, *, loop: asyncio.AbstractEventLoop) -> None:
@@ -727,7 +694,7 @@ async def change_presence(
         self.afk = afk
         self.idle_since = since
 
-    async def request_lazy_guild(
+    async def guild_subscribe(
         self,
         guild_id: Snowflake,
         *,
@@ -762,6 +729,17 @@ async def request_lazy_guild(
         _log.debug('Subscribing to guild %s with payload %s', guild_id, payload['d'])
         await self.send_as_json(payload)
 
+    async def bulk_guild_subscribe(self, subscriptions: BulkGuildSubscribePayload) -> None:
+        payload = {
+            'op': self.BULK_GUILD_SUBSCRIBE,
+            'd': {
+                'subscriptions': subscriptions,
+            },
+        }
+
+        _log.debug('Subscribing to guilds with payload %s', payload['d'])
+        await self.send_as_json(payload)
+
     async def request_chunks(
         self,
         guild_ids: List[Snowflake],
@@ -821,44 +799,6 @@ async def call_connect(self, channel_id: Snowflake):
         _log.debug('Requesting call connect for channel %s.', channel_id)
         await self.send_as_json(payload)
 
-    async def request_commands(
-        self,
-        guild_id: Snowflake,
-        type: int,
-        *,
-        nonce: Optional[str] = None,
-        limit: Optional[int] = None,
-        applications: Optional[bool] = None,
-        offset: int = 0,
-        query: Optional[str] = None,
-        command_ids: Optional[List[Snowflake]] = None,
-        application_id: Optional[Snowflake] = None,
-    ) -> None:
-        payload = {
-            'op': self.REQUEST_COMMANDS,
-            'd': {
-                'guild_id': str(guild_id),
-                'type': type,
-            },
-        }
-
-        if nonce is not None:
-            payload['d']['nonce'] = nonce
-        if applications is not None:
-            payload['d']['applications'] = applications
-        if limit is not None and limit != 25:
-            payload['d']['limit'] = limit
-        if offset:
-            payload['d']['offset'] = offset
-        if query is not None:
-            payload['d']['query'] = query
-        if command_ids is not None:
-            payload['d']['command_ids'] = command_ids
-        if application_id is not None:
-            payload['d']['application_id'] = str(application_id)
-
-        await self.send_as_json(payload)
-
     async def search_recent_members(
         self, guild_id: Snowflake, query: str = '', *, after: Optional[Snowflake] = None, nonce: Optional[str] = None
     ) -> None: