feat: add member and guild tombstones

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Fixes
 
+- Preserve guild and member history with source-attributed tombstones, explicit restore semantics, omission-safe member refreshes, and revision-aware Git-share merges.
 - Keep attachment fetches compatible with three allowed Discord CDN redirects and injected HTTP transports while preserving final-response host validation.
 - Reject malformed tombstone timestamps before routine incremental snapshot imports mutate the archive.
 

README.md

@@ -293,7 +293,7 @@ Each channel crawl also has a bounded runtime budget, so a pathological channel
 Retryable failures and unavailable-channel markers are tracked per channel; stale unavailable markers are cleared after a later successful crawl, and marker cleanup is best-effort so one missing local sync-state row cannot crash the run.
 Full sync member refresh is best-effort and currently gives up after five minutes without a caller-supplied deadline, so message sync completion is not held hostage by a slow guild member crawl.
 When the archive is already complete, `sync --full` now reuses the stored backlog markers and limits steady-state refresh to live top-level channels plus active threads instead of revisiting every stored archived thread.
-If a guild already has a local member snapshot, routine syncs reuse it and skip another full member crawl until that snapshot ages out.
+If a guild already has a local member snapshot, routine syncs reuse it and skip another full member crawl until that snapshot ages out. A completed refresh merges the members it observed; an omitted member is retained unless Discord sends an explicit member-remove event.
 
 ### `tail`
 
@@ -640,7 +640,7 @@ discrawl subscribe --stale-after 15m https://github.com/example/discord-archive.
 discrawl subscribe --no-auto-update https://github.com/example/discord-archive.git
 ```
 
-Once `share.remote` is configured, read commands auto-fetch and import when the last share check is older than `share.stale_after` (default `15m`). Imports are planned from crawlkit shard fingerprints, with a Git-object fallback for older manifests, so routine updates normally read only changed canonical shards, preserve destination-only cache rows, and avoid rebuilding message FTS. `discrawl update` runs the same safe merge manually. Removed shards or incompatible table changes keep the database untouched and require `discrawl update --force`; historical restores require `update --force --ref <tag-or-commit>`. `discrawl sync` does not auto-import the share unless `--update=auto` or `--update=force` is provided.
+Once `share.remote` is configured, read commands auto-fetch and import when the last share check is older than `share.stale_after` (default `15m`). Imports are planned from crawlkit shard fingerprints, with a Git-object fallback for older manifests, so routine updates normally read only changed canonical shards, preserve destination-only cache rows, and avoid rebuilding message FTS. Guild and member rows carry explicit deletion metadata; their merges keep newer revisions, prefer tombstones at equal revisions, and allow later live revisions to restore them. `discrawl update` runs the same safe merge manually. Removed shards or incompatible table changes keep the database untouched and require `discrawl update --force`; tombstone-compatible guild/member schema changes remain merge-safe, while historical restores require `update --force --ref <tag-or-commit>`. `discrawl sync` does not auto-import the share unless `--update=auto` or `--update=force` is provided.
 
 Hybrid mode is supported too: keep normal Discord credentials configured and set `share.remote`. `discrawl sync --update=auto` and `discrawl messages --sync` safely merge the Git snapshot first, usually as a changed-shard delta, then use live Discord for latest-message deltas. `discrawl sync --update=force` intentionally replaces public snapshot tables before live sync. Use `sync --all-channels` or `sync --full` when you want a broader live repair/backfill pass.
 

internal/cli/cloud_commands.go

@@ -460,7 +460,7 @@ var discrawlGuildColumns = []string{"guild_id", "name", "updated_at"}
 const discrawlGuildExportSQL = `
 select id as guild_id, name, updated_at
 from guilds
-where id != '@me'
+where id != '@me' and deleted_at is null
 order by id`
 
 var discrawlChannelColumns = []string{"channel_id", "guild_id", "name", "type", "parent_id", "updated_at"}
@@ -476,7 +476,7 @@ var discrawlMemberColumns = []string{"guild_id", "user_id", "username", "display
 const discrawlMemberExportSQL = `
 select guild_id, user_id, username, coalesce(nullif(display_name, ''), nullif(nick, ''), username) as display_name, updated_at
 from members
-where guild_id != '@me'
+where guild_id != '@me' and deleted_at is null
 order by guild_id, user_id`
 
 var discrawlMessageColumns = []string{"message_id", "channel_id", "guild_id", "author_id", "author_username", "content", "created_at", "edited_at"}

internal/discord/client.go

@@ -22,6 +22,11 @@ type EventHandler interface {
 	OnMemberDelete(context.Context, string, string) error
 }
 
+type guildEventHandler interface {
+	OnGuildUpsert(context.Context, *discordgo.Guild) error
+	OnGuildDelete(context.Context, *discordgo.Guild) error
+}
+
 type tailGuildFilter interface {
 	TailAllowsGuild(string) bool
 }
@@ -603,6 +608,51 @@ func (c *Client) Tail(ctx context.Context, handler EventHandler) error {
 			before,
 		))
 	})
+	addHandler(func(_ *discordgo.Session, evt *discordgo.GuildCreate) {
+		guildHandler, ok := handler.(guildEventHandler)
+		if !ok {
+			return
+		}
+		var guild *discordgo.Guild
+		if evt != nil {
+			guild = evt.Guild
+		}
+		c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newGuildTailTask(
+			"GUILD_CREATE",
+			func(taskCtx context.Context) error { return guildHandler.OnGuildUpsert(taskCtx, guild) },
+			guild,
+		))
+	})
+	addHandler(func(_ *discordgo.Session, evt *discordgo.GuildUpdate) {
+		guildHandler, ok := handler.(guildEventHandler)
+		if !ok {
+			return
+		}
+		var guild *discordgo.Guild
+		if evt != nil {
+			guild = evt.Guild
+		}
+		c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newGuildTailTask(
+			"GUILD_UPDATE",
+			func(taskCtx context.Context) error { return guildHandler.OnGuildUpsert(taskCtx, guild) },
+			guild,
+		))
+	})
+	addHandler(func(_ *discordgo.Session, evt *discordgo.GuildDelete) {
+		guildHandler, ok := handler.(guildEventHandler)
+		if !ok {
+			return
+		}
+		var guild *discordgo.Guild
+		if evt != nil {
+			guild = evt.Guild
+		}
+		c.enqueueTailTask(tailCtx, orderedWorkCh, fatal, newGuildTailTask(
+			"GUILD_DELETE",
+			func(taskCtx context.Context) error { return guildHandler.OnGuildDelete(taskCtx, guild) },
+			guild,
+		))
+	})
 	addHandler(func(_ *discordgo.Session, evt *discordgo.GuildMemberAdd) {
 		var member *discordgo.Member
 		if evt != nil {
@@ -1297,6 +1347,24 @@ func newChannelTailTask(
 	return task
 }
 
+func newGuildTailTask(
+	eventType string,
+	run func(context.Context) error,
+	guilds ...*discordgo.Guild,
+) tailTask {
+	task := tailTask{
+		eventType:    eventType,
+		failureClass: tailFailureClassOrdered,
+		run:          run,
+	}
+	for _, guild := range guilds {
+		if guild != nil {
+			setTailTaskID(&task.guildID, guild.ID)
+		}
+	}
+	return task
+}
+
 func newMemberTailTask(
 	eventType string,
 	run func(context.Context) error,

internal/share/merge.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -62,7 +63,7 @@ func MergeIfChanged(ctx context.Context, s *store.Store, opts Options) (Manifest
 		}
 	}
 	plan := snapshot.PlanMergeImport(snapshotManifest(previous), snapshotManifest(manifest))
-	plan, err = shareMergePlan(plan, allowEventMerge)
+	plan, err = shareMergePlan(plan, snapshotManifest(previous), allowEventMerge)
 	if err != nil {
 		if markErr := MarkReplacementPending(ctx, s, manifest, err.Error()); markErr != nil {
 			return Manifest{}, false, errors.Join(err, markErr)
@@ -72,7 +73,7 @@ func MergeIfChanged(ctx context.Context, s *store.Store, opts Options) (Manifest
 	return importMergePlan(ctx, s, opts, previous, manifest, plan)
 }
 
-func shareMergePlan(plan snapshot.ImportPlan, allowEventMerge bool) (snapshot.ImportPlan, error) {
+func shareMergePlan(plan snapshot.ImportPlan, previous snapshot.Manifest, allowEventMerge bool) (snapshot.ImportPlan, error) {
 	if plan.Full {
 		return snapshot.ImportPlan{}, &ReplacementRequiredError{Reason: plan.Reason}
 	}
@@ -104,6 +105,14 @@ func shareMergePlan(plan snapshot.ImportPlan, allowEventMerge bool) (snapshot.Im
 			continue
 		}
 		if tablePlan.Mode == snapshot.TableImportReplace {
+			if (tablePlan.Table.Name == "guilds" || tablePlan.Table.Name == "members") &&
+				tablePlan.Reason == "columns changed" &&
+				isTombstoneColumnAddition(manifestTable(previous, tablePlan.Table.Name).Columns, tablePlan.Table.Columns) {
+				tablePlan.Mode = snapshot.TableImportFiles
+				tablePlan.Reason = "merge tombstone-aware entity rows"
+				out.Tables = append(out.Tables, tablePlan)
+				continue
+			}
 			replacements = append(replacements, tablePlan.Table.Name)
 			continue
 		}
@@ -116,6 +125,43 @@ func shareMergePlan(plan snapshot.ImportPlan, allowEventMerge bool) (snapshot.Im
 	return out, nil
 }
 
+func manifestTable(manifest snapshot.Manifest, name string) snapshot.TableManifest {
+	for _, table := range manifest.Tables {
+		if table.Name == name {
+			return table
+		}
+	}
+	return snapshot.TableManifest{}
+}
+
+func isTombstoneColumnAddition(previous, current []string) bool {
+	if len(current) != len(previous)+3 {
+		return false
+	}
+	base := make([]string, 0, len(previous))
+	tombstones := map[string]bool{
+		"deleted_at":      false,
+		"deletion_source": false,
+		"deletion_reason": false,
+	}
+	for _, column := range current {
+		if _, ok := tombstones[column]; ok {
+			if tombstones[column] {
+				return false
+			}
+			tombstones[column] = true
+			continue
+		}
+		base = append(base, column)
+	}
+	for _, found := range tombstones {
+		if !found {
+			return false
+		}
+	}
+	return slices.Equal(previous, base)
+}
+
 func eventTablesEmpty(ctx context.Context, s *store.Store) (bool, error) {
 	var rows int
 	if err := s.DB().QueryRowContext(ctx, `select (select count(*) from message_events) + (select count(*) from mention_events)`).Scan(&rows); err != nil {