Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
- Report module build metadata for source-installed binaries instead of a stale hard-coded release version.
- Keep Discord Gateway tails fail-closed while durably spooling and replaying timed-out message creates, updates, and deletes. Thanks @hannesrudolph.
- Reject attachment redirects whose final URL leaves Discord's allowlisted CDN hosts. Thanks @GrantTheAssistant.
- Reject malformed message tombstone timestamps before either SQLite snapshot import path mutates the archive. Thanks @GrantTheAssistant.

## 0.11.5 - 2026-07-09

Expand Down
36 changes: 35 additions & 1 deletion internal/share/share.go
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,13 @@ func Import(ctx context.Context, s *store.Store, opts Options) (Manifest, error)
})
},
Filter: func(table string, row map[string]any) (bool, error) {
return !isDirectMessageSnapshotRow(table, row), nil
if isDirectMessageSnapshotRow(table, row) {
return false, nil
}
if err := validateSnapshotRow(table, row); err != nil {
return false, err
}
Comment on lines +506 to +508

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Validate tombstones on the incremental path too

This installs the new validator for full replacement imports, but the normal MergeIfChanged path still uses snapshot.ImportIncremental with a filter that only skips DMs, and importMergeSnapshotRow/upsertMergeSnapshotRow do not call validateSnapshotRow. When a changed-tail snapshot is applied through MergeIfChanged, a malformed or non-string messages.deleted_at can still be written to the archive before any error, so the main supported update path remains unprotected. Please run the same validation from the incremental filter or import row hook as well.

Useful? React with 👍 / 👎.

return true, nil
},
BeforeImport: func(ctx context.Context, tx *sql.Tx) error {
var err error
Expand Down Expand Up @@ -1659,6 +1665,9 @@ func importTableFile(ctx context.Context, stmt *sql.Stmt, repoPath string, table
if isDirectMessageSnapshotRow(table.Name, row) {
continue
}
if err := validateSnapshotRow(table.Name, row); err != nil {
return count, fmt.Errorf("validate %s: %w", rel, err)
}
values := make([]any, len(columns))
for i, column := range columns {
values[i] = importValue(row[column])
Expand All @@ -1671,6 +1680,31 @@ func importTableFile(ctx context.Context, stmt *sql.Stmt, repoPath string, table
return count, nil
}

func validateSnapshotRow(table string, row map[string]any) error {
if table != "messages" {
return nil
}
raw, ok := row["deleted_at"]
if !ok || raw == nil {
return nil
}
value, ok := raw.(string)
if !ok {
return errors.New("messages.deleted_at must be a string or null")
}
value = strings.TrimSpace(value)
if value == "" {
row["deleted_at"] = nil
return nil
}
parsed, err := time.Parse(time.RFC3339Nano, value)
if err != nil {
return fmt.Errorf("messages.deleted_at must be RFC3339: %w", err)
}
row["deleted_at"] = parsed.UTC().Format(time.RFC3339Nano)
return nil
}

func repairImportedGuildIDs(ctx context.Context, tx *sql.Tx) error {
repairs := []struct {
table string
Expand Down
39 changes: 39 additions & 0 deletions internal/share/share_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2275,6 +2275,45 @@ func TestLegacyManifestFileImportAndEmbeddingDecodeErrors(t *testing.T) {
require.NoError(t, tx.Rollback())
}

func TestValidateSnapshotRowRejectsMalformedDeletedAtBeforeImport(t *testing.T) {
require.NoError(t, validateSnapshotRow("messages", map[string]any{"deleted_at": nil}))
blank := map[string]any{"deleted_at": " "}
require.NoError(t, validateSnapshotRow("messages", blank))
require.Nil(t, blank["deleted_at"])
require.NoError(t, validateSnapshotRow("messages", map[string]any{"deleted_at": "2026-07-14T12:00:00.123456789Z"}))
padded := map[string]any{"deleted_at": " 2026-07-14T12:00:00Z "}
require.NoError(t, validateSnapshotRow("messages", padded))
require.Equal(t, "2026-07-14T12:00:00Z", padded["deleted_at"])
require.ErrorContains(t, validateSnapshotRow("messages", map[string]any{"deleted_at": "not-a-timestamp"}), "must be RFC3339")
require.ErrorContains(t, validateSnapshotRow("messages", map[string]any{"deleted_at": json.Number("123")}), "must be a string or null")
require.NoError(t, validateSnapshotRow("guilds", map[string]any{"deleted_at": "not-a-timestamp"}))

ctx := context.Background()
s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db"))
require.NoError(t, err)
defer func() { _ = s.Close() }()
repo := t.TempDir()
rel := filepath.ToSlash(filepath.Join("tables", "messages", "tombstones.jsonl.gz"))
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(repo, filepath.FromSlash(rel))), 0o755))
writeGzipJSONLines(t, filepath.Join(repo, filepath.FromSlash(rel)), []string{
`{"id":"m1","guild_id":"g1","channel_id":"c1","author_id":null,"message_type":0,"created_at":"2026-07-14T12:00:00Z","edited_at":null,"deleted_at":null,"content":"one","normalized_content":"one","reply_to_message_id":null,"pinned":0,"has_attachments":0,"raw_json":"{}","updated_at":"2026-07-14T12:00:00Z"}`,
`{"id":"m2","guild_id":"g1","channel_id":"c1","author_id":null,"message_type":0,"created_at":"2026-07-14T12:00:00Z","edited_at":null,"deleted_at":"not-a-timestamp","content":"two","normalized_content":"two","reply_to_message_id":null,"pinned":0,"has_attachments":0,"raw_json":"{}","updated_at":"2026-07-14T12:00:00Z"}`,
})
tx, err := s.DB().BeginTx(ctx, nil)
require.NoError(t, err)
err = importTable(ctx, tx, Options{RepoPath: repo}, TableManifest{
Name: "messages", File: rel,
Columns: []string{"id", "guild_id", "channel_id", "author_id", "message_type", "created_at", "edited_at", "deleted_at", "content", "normalized_content", "reply_to_message_id", "pinned", "has_attachments", "raw_json", "updated_at"},
})
require.ErrorContains(t, err, "messages.deleted_at must be RFC3339")
var count int
require.NoError(t, tx.QueryRowContext(ctx, `select count(*) from messages`).Scan(&count))
require.Equal(t, 1, count)
require.NoError(t, tx.Rollback())
require.NoError(t, s.DB().QueryRowContext(ctx, `select count(*) from messages`).Scan(&count))
require.Zero(t, count)
}

func TestImportEmbeddingsRejectsUnsafeManifestFiles(t *testing.T) {
t.Parallel()

Expand Down