Skip to content

Commit

Permalink
better memory handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@Klemek
Klemek committed Apr 7, 2021
1 parent 45d56a3 commit 562fd51
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 112 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ __pycache__
error_*
*.log
/logs/
.vscode
16 changes: 15 additions & 1 deletion src/logs/guild_logs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,15 @@ def __init__(self, guild: discord.Guild):
self.channels = {}
self.locked = False

def __enter__(self):
return self

def __exit__(self, type, value, tb):
del self.channels
del self.guild
if self.locked:
self.unlock()

def dict(self) -> dict:
return {id: self.channels[id].dict() for id in self.channels}

Expand All @@ -77,7 +86,8 @@ def lock(self) -> bool:
def unlock(self):
self.locked = False
current_analysis_lock.acquire()
current_analysis.remove(self.log_file)
if self.log_file in current_analysis:
current_analysis.remove(self.log_file)
current_analysis_lock.release()

async def load(
Expand Down Expand Up @@ -111,6 +121,7 @@ async def load(
await code_message(progress, "Reading saved history (2/4)...")
t0 = datetime.now()
json_data = gzip.decompress(gziped_data)
del gziped_data
logging.info(
f"log {self.guild.id} > gzip decompress in {delta(t0):,}ms"
)
Expand All @@ -119,6 +130,7 @@ async def load(
await code_message(progress, "Reading saved history (3/4)...")
t0 = datetime.now()
channels = json.loads(json_data)
del json_data
logging.info(f"log {self.guild.id} > json parse in {delta(t0):,}ms")
if self.check_cancelled():
return CANCELLED, 0
Expand Down Expand Up @@ -267,6 +279,7 @@ async def load(
)
t0 = datetime.now()
gziped_data = gzip.compress(json_data)
del json_data
logging.info(
f"log {self.guild.id} > gzip in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
)
Expand All @@ -279,6 +292,7 @@ async def load(
t0 = datetime.now()
with open(self.log_file, mode="wb") as f:
f.write(gziped_data)
del gziped_data
logging.info(
f"log {self.guild.id} > saved in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
)
Expand Down
223 changes: 112 additions & 111 deletions src/scanners/scanner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,133 +36,134 @@ async def compute(
):
args = list(args)
guild = message.guild
logs = GuildLogs(guild)

# If "%cmd help" redirect to "%help cmd"
if "help" in args:
await client.bot.help(client, message, "help", args[0])
return

# check args validity
str_channel_mentions = [str(channel.id) for channel in message.channel_mentions]
str_mentions = [str(member.id) for member in message.mentions]
for i, arg in enumerate(args[1:]):
if re.match(r"^<@!?\d+>$", arg):
arg = arg[3:-1] if "!" in arg else arg[2:-1]
elif re.match(r"^<#!?\d+>$", arg):
arg = arg[3:-1] if "!" in arg else arg[2:-1]
if (
arg not in self.valid_args + ["me", "here", "fast", "fresh"]
and (not arg.isdigit() or not self.has_digit_args)
and arg not in str_channel_mentions
and arg not in str_mentions
):
await message.channel.send(
f"Unrecognized argument: `{arg}`", reference=message
)
with GuildLogs(guild) as logs:
# If "%cmd help" redirect to "%help cmd"
if "help" in args:
await client.bot.help(client, message, "help", args[0])
return

# check args validity
str_channel_mentions = [
str(channel.id) for channel in message.channel_mentions
]
str_mentions = [str(member.id) for member in message.mentions]
for i, arg in enumerate(args[1:]):
if re.match(r"^<@!?\d+>$", arg):
arg = arg[3:-1] if "!" in arg else arg[2:-1]
elif re.match(r"^<#!?\d+>$", arg):
arg = arg[3:-1] if "!" in arg else arg[2:-1]
if (
arg not in self.valid_args + ["me", "here", "fast", "fresh"]
and (not arg.isdigit() or not self.has_digit_args)
and arg not in str_channel_mentions
and arg not in str_mentions
):
await message.channel.send(
f"Unrecognized argument: `{arg}`", reference=message
)
return

# Get selected channels or all of them if no channel arguments
self.channels = no_duplicate(message.channel_mentions)

# transform the "here" arg
if "here" in args:
self.channels += [message.channel]

self.full = len(self.channels) == 0
if self.full:
self.channels = guild.text_channels

# Get selected members
self.members = no_duplicate(message.mentions)
self.raw_members = no_duplicate(message.raw_mentions)

# transform the "me" arg
if "me" in args:
self.members += [message.author]
self.raw_members += [message.author.id]

if not await self.init(message, *args):
return

# Get selected channels or all of them if no channel arguments
self.channels = no_duplicate(message.channel_mentions)

# transform the "here" arg
if "here" in args:
self.channels += [message.channel]

self.full = len(self.channels) == 0
if self.full:
self.channels = guild.text_channels

# Get selected members
self.members = no_duplicate(message.mentions)
self.raw_members = no_duplicate(message.raw_mentions)

# transform the "me" arg
if "me" in args:
self.members += [message.author]
self.raw_members += [message.author.id]

if not await self.init(message, *args):
return

# Start computing data
async with message.channel.typing():
progress = await message.channel.send(
"```Starting analysis...```",
reference=message,
allowed_mentions=discord.AllowedMentions.none(),
)
total_msg, total_chan = await logs.load(
progress, self.channels, fast="fast" in args, fresh="fresh" in args
)
if total_msg == CANCELLED:
await message.channel.send(
"Operation cancelled by user",
# Start computing data
async with message.channel.typing():
progress = await message.channel.send(
"```Starting analysis...```",
reference=message,
allowed_mentions=discord.AllowedMentions.none(),
)
elif total_msg == ALREADY_RUNNING:
await message.channel.send(
"An analysis is already running on this server, please be patient.",
reference=message,
total_msg, total_chan = await logs.load(
progress, self.channels, fast="fast" in args, fresh="fresh" in args
)
else:
self.msg_count = 0
self.total_msg = 0
self.chan_count = 0
t0 = datetime.now()
for channel in self.channels:
if channel.id in logs.channels:
channel_logs = logs.channels[channel.id]
count = sum(
[
self.compute_message(channel_logs, message_log)
for message_log in channel_logs.messages
]
)
self.total_msg += len(channel_logs.messages)
self.msg_count += count
self.chan_count += 1 if count > 0 else 0
logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms")
if self.total_msg == 0:
if total_msg == CANCELLED:
await message.channel.send(
"Operation cancelled by user",
reference=message,
)
elif total_msg == ALREADY_RUNNING:
await message.channel.send(
"There are no messages found matching the filters",
"An analysis is already running on this server, please be patient.",
reference=message,
)
else:
await progress.edit(content="```Computing results...```")
# Display results
self.msg_count = 0
self.total_msg = 0
self.chan_count = 0
t0 = datetime.now()
results = self.get_results(
get_intro(
self.intro_context,
self.full,
self.channels,
self.members,
self.msg_count,
self.chan_count,
for channel in self.channels:
if channel.id in logs.channels:
channel_logs = logs.channels[channel.id]
count = sum(
[
self.compute_message(channel_logs, message_log)
for message_log in channel_logs.messages
]
)
self.total_msg += len(channel_logs.messages)
self.msg_count += count
self.chan_count += 1 if count > 0 else 0
logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms")
if self.total_msg == 0:
await message.channel.send(
"There are no messages found matching the filters",
reference=message,
)
)
logging.info(f"scan {guild.id} > results in {delta(t0):,}ms")
response = ""
first = True
for r in results:
if len(response + "\n" + r) > 2000:
else:
await progress.edit(content="```Computing results...```")
# Display results
t0 = datetime.now()
results = self.get_results(
get_intro(
self.intro_context,
self.full,
self.channels,
self.members,
self.msg_count,
self.chan_count,
)
)
logging.info(f"scan {guild.id} > results in {delta(t0):,}ms")
response = ""
first = True
for r in results:
if len(response + "\n" + r) > 2000:
await message.channel.send(
response,
reference=message if first else None,
allowed_mentions=discord.AllowedMentions.none(),
)
first = False
response = ""
response += "\n" + r
if len(response) > 0:
await message.channel.send(
response,
reference=message if first else None,
allowed_mentions=discord.AllowedMentions.none(),
)
first = False
response = ""
response += "\n" + r
if len(response) > 0:
await message.channel.send(
response,
reference=message if first else None,
allowed_mentions=discord.AllowedMentions.none(),
)
# Delete custom progress message
await progress.delete()
# Delete custom progress message
await progress.delete()

@abstractmethod
async def init(self, message: discord.Message, *args: str) -> bool:
Expand Down

0 comments on commit 562fd51

Please sign in to comment.