diff --git a/README.md b/README.md index bbe3497..a8b6078 100644 --- a/README.md +++ b/README.md @@ -125,11 +125,14 @@ python3 src/main.py ## Changelog +* **v1.17** + * compliency with 30 days data keeping policy and data encryption + * improvements and bug fix * **v1.16** * `%freq graph` graph hours frequency along the week * uses discord new time format * `%freq` now shows quietest day of week and hour of day - * improvments and bug fix + * improvements and bug fix * **v1.15** * `nsfw:allow/only` filter nsfw channels * `%find` can use regexes diff --git a/requirements.txt b/requirements.txt index 9075d8f..384a9d2 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -discord.py==1.7.0 -python-dotenv==0.15.0 -python-dateutil==2.8.1 -git+git://github.com/Klemek/miniscord.git -matplotlib -cryptography \ No newline at end of file +discord.py^=1.7.0 +python-dotenv^=0.15.0 +python-dateutil^=2.8.1 +matplotlib^=3.4.2 +cryptography^=2.8 +git+git://github.com/Klemek/miniscord.git \ No newline at end of file diff --git a/src/logs/guild_logs.py b/src/logs/guild_logs.py index 31151ea..cbcefb7 100644 --- a/src/logs/guild_logs.py +++ b/src/logs/guild_logs.py @@ -9,6 +9,7 @@ import asyncio import threading from dotenv import load_dotenv +from cryptography.fernet import Fernet from . import ChannelLogs from utils import code_message, delta, deltas @@ -23,7 +24,7 @@ load_dotenv() LOG_DIR = os.getenv("LOG_DIR", "logs") -LOG_EXT = os.getenv("LOG_DIR", ".logz") +LOG_EXT = os.getenv("LOG_EXT", ".logz") CRYPT_KEY = os.getenv("CRYPT_KEY", "") # 5 minutes, assume 'fast' arg @@ -130,29 +131,41 @@ async def load( channels = {} try: last_time = os.path.getmtime(self.log_file) - gziped_data = None - await code_message(progress, "Reading saved history (1/4)...") + encrypted_data = None + await code_message(progress, "Reading saved history (1/5)...") t0 = datetime.now() with open(self.log_file, mode="rb") as f: - gziped_data = f.read() + encrypted_data = f.read() logging.info(f"log {self.guild.id} > read in {delta(t0):,}ms") if self.check_cancelled(): return CANCELLED, 0 - await code_message(progress, "Reading saved history (2/4)...") + await code_message(progress, "Reading saved history (2/5)...") + if CRYPT_KEY == "" or CRYPT_KEY is None: + gziped_data = encrypted_data + try: + t0 = datetime.now() + fernet = Fernet(CRYPT_KEY) + gziped_data = fernet.decrypt(encrypted_data) + logging.info(f"log {self.guild.id} > decrypted in {delta(t0):,}ms") + except: + gziped_data = encrypted_data + if self.check_cancelled(): + return CANCELLED, 0 + await code_message(progress, "Reading saved history (3/5)...") t0 = datetime.now() json_data = gzip.decompress(gziped_data) del gziped_data logging.info(f"log {self.guild.id} > gzip decompress in {delta(t0):,}ms") if self.check_cancelled(): return CANCELLED, 0 - await code_message(progress, "Reading saved history (3/4)...") + await code_message(progress, "Reading saved history (4/5)...") t0 = datetime.now() channels = json.loads(json_data) del json_data logging.info(f"log {self.guild.id} > json parse in {delta(t0):,}ms") if self.check_cancelled(): return CANCELLED, 0 - await code_message(progress, "Reading saved history (4/4)...") + await code_message(progress, "Reading saved history (5/5)...") t0 = datetime.now() self.channels = { int(id): ChannelLogs(channels[id], self) for id in channels @@ -289,7 +302,7 @@ async def load( return CANCELLED, 0 await code_message( progress, - f"Saving history (1/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", + f"Saving history (1/4)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", ) t0 = datetime.now() json_data = bytes(json.dumps(self.dict()), "utf-8") @@ -300,7 +313,7 @@ async def load( return CANCELLED, 0 await code_message( progress, - f"Saving history (2/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", + f"Saving history (2/4)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", ) t0 = datetime.now() gziped_data = gzip.compress(json_data) @@ -312,12 +325,28 @@ async def load( return CANCELLED, 0 await code_message( progress, - f"Saving history (3/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", + f"Saving history (3/4)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", + ) + if CRYPT_KEY == "" or CRYPT_KEY is None: + encrypted_data = gziped_data + try: + t0 = datetime.now() + fernet = Fernet(CRYPT_KEY) + encrypted_data = fernet.encrypt(gziped_data) + logging.info(f"log {self.guild.id} > encrypted in {delta(t0):,}ms -> {len(gziped_data) / deltas(t0):,.3f} b/s") + except: + encrypted_data = gziped_data + if self.check_cancelled(): + return CANCELLED, 0 + await code_message( + progress, + f"Saving history (4/4)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", ) t0 = datetime.now() with open(self.log_file, mode="wb") as f: - f.write(gziped_data) + f.write(encrypted_data) del gziped_data + del encrypted_data logging.info( f"log {self.guild.id} > saved in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s" ) diff --git a/src/main.py b/src/main.py index 9e9af55..6549b02 100644 --- a/src/main.py +++ b/src/main.py @@ -18,7 +18,7 @@ bot = Bot( "Discord Analyst", - "1.16.2", + "1.17.0", alias="%", ) diff --git a/src/tools/convert_logs.py b/src/tools/convert_logs.py new file mode 100644 index 0000000..132cc04 --- /dev/null +++ b/src/tools/convert_logs.py @@ -0,0 +1,26 @@ +import os +import os.path +from dotenv import load_dotenv +from cryptography.fernet import Fernet + +load_dotenv() + +LOG_DIR = os.getenv("LOG_DIR", "logs") +LOG_EXT = os.getenv("LOG_DIR", ".logz") +CRYPT_KEY = os.getenv("CRYPT_KEY", "") + +fernet = Fernet(CRYPT_KEY) + +for item in os.listdir(LOG_DIR): + if item.endswith(LOG_EXT): + path = os.path.join(LOG_DIR, item) + data = None + with open(path, mode="rb") as f: + data = f.read() + try: + fernet.decrypt(data) + print(f"{item} already encrypted") + except: + with open(path, mode="wb") as f: + f.write(fernet.encrypt(data)) + print(f"{item} was encrypted") \ No newline at end of file