Skip to content

Commit

Permalink
fix: Remove msgspec dependency
Browse files Browse the repository at this point in the history
The builtin json module works just fine. It is a bit slower for large
files, but this will be fixed with the upcoming rust based logfile
parser.
  • Loading branch information
rumpelsepp committed Nov 28, 2024
1 parent e2c924e commit 06a8b6b
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 49 deletions.
1 change: 0 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ dependencies = [
"boltons>=24.1.0",
"construct >=2.10,<3.0",
"more-itertools >=10.3.0,<11.0.0",
"msgspec >=0.11,<0.19",
"platformdirs >=2.6,<5.0",
"pydantic >=2.0,<3.0",
"tabulate >=0.9",
Expand Down
5 changes: 2 additions & 3 deletions src/gallia/cli/hr.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,14 @@
# SPDX-License-Identifier: Apache-2.0

import argparse
import json
import os
import signal
import sys
from itertools import islice
from pathlib import Path
from typing import cast

import msgspec

from gallia import exitcodes
from gallia.log import ColorMode, PenlogPriority, PenlogReader, resolve_color_mode

Expand Down Expand Up @@ -89,7 +88,7 @@ def _main() -> int:
def main() -> None:
try:
sys.exit(_main())
except (msgspec.DecodeError, msgspec.ValidationError) as e:
except json.JSONDecodeError as e:
print(f"invalid file format: {e}", file=sys.stderr)
sys.exit(exitcodes.DATAERR)
# BrokenPipeError appears when stuff is piped to | head.
Expand Down
7 changes: 4 additions & 3 deletions src/gallia/command/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# SPDX-License-Identifier: Apache-2.0

import asyncio
import dataclasses
import json
import os
import os.path
Expand All @@ -19,7 +20,6 @@
from tempfile import gettempdir
from typing import Any, Protocol, Self, cast

import msgspec
from pydantic import ConfigDict, field_serializer, model_validator

from gallia import exitcodes
Expand Down Expand Up @@ -49,15 +49,16 @@ class HookVariant(Enum):
POST = "post"


class RunMeta(msgspec.Struct):
@dataclasses.dataclass
class RunMeta:
command: str
start_time: str
end_time: str
exit_code: int
config: MutableMapping[str, Any]

def json(self) -> str:
return msgspec.json.encode(self).decode()
return json.dumps(dataclasses.asdict(self))


logger = get_logger(__name__)
Expand Down
44 changes: 26 additions & 18 deletions src/gallia/log.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@
from __future__ import annotations

import atexit
import dataclasses
import datetime
import gzip
import io
import json
import logging
import mmap
import os
Expand All @@ -18,15 +20,13 @@
import time
import traceback
from collections.abc import Iterator
from dataclasses import dataclass
from enum import Enum, IntEnum, unique
from logging.handlers import QueueHandler, QueueListener
from pathlib import Path
from queue import Queue
from types import TracebackType
from typing import TYPE_CHECKING, Any, BinaryIO, Self, TextIO, TypeAlias, cast

import msgspec
import zstandard

if TYPE_CHECKING:
Expand Down Expand Up @@ -327,12 +327,14 @@ def add_zst_log_handler(
return zstd_handler


class _PenlogRecordV2(msgspec.Struct, omit_defaults=True, tag=2, tag_field="version"):
@dataclasses.dataclass
class _PenlogRecordV2:
module: str
host: str
data: str
datetime: str
priority: int
version: int
tags: list[str] | None = None
line: str | None = None
stacktrace: str | None = None
Expand Down Expand Up @@ -417,7 +419,7 @@ def _format_record( # noqa: PLR0913
return msg


@dataclass
@dataclasses.dataclass
class PenlogRecord:
module: str
host: str
Expand Down Expand Up @@ -459,20 +461,26 @@ def parse_json(cls, data: bytes) -> Self:
if data.startswith(b"<"):
data = data[data.index(b">") + 1 :]

record = msgspec.json.decode(data, type=_PenlogRecordV2)
record = json.loads(data.decode())
if (v := record["version"]) != 2:
raise json.JSONDecodeError(f"invalid log record version {v}", data.decode(), 0)

return cls(
module=record.module,
host=record.host,
data=record.data,
datetime=datetime.datetime.fromisoformat(record.datetime),
priority=PenlogPriority(record.priority),
tags=record.tags,
line=record.line,
stacktrace=record.stacktrace,
_python_level_no=record._python_level_no,
_python_level_name=record._python_level_name,
_python_func_name=record._python_func_name,
module=record["module"],
host=record["host"],
data=record["data"],
datetime=datetime.datetime.fromisoformat(record["datetime"]),
priority=PenlogPriority(record["priority"]),
tags=record["tags"] if "tags" in record else None,
line=record["line"] if "line" in record else None,
stacktrace=record["stacktrace"] if "stacktrace" in record else None,
_python_level_no=record["_python_level_no"] if "_python_level_no" in record else None,
_python_level_name=record["_python_level_name"]
if "_python_level_name" in record
else None,
_python_func_name=record["_python_func_name"]
if "_python_func_name" in record
else None,
)

def to_log_record(self) -> logging.LogRecord:
Expand Down Expand Up @@ -692,9 +700,9 @@ def format(self, record: logging.LogRecord) -> str:
_python_level_no=record.levelno,
_python_level_name=record.levelname,
_python_func_name=record.funcName,
version=2,
)

return msgspec.json.encode(penlog_record).decode()
return json.dumps(dataclasses.asdict(penlog_record))


class _ConsoleFormatter(logging.Formatter):
Expand Down
24 changes: 0 additions & 24 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 06a8b6b

Please sign in to comment.