Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
### Fixes

- Kept resumed `sync --full` backfills from moving channel latest-message checkpoints backward, avoiding duplicate head recrawls on large interrupted channels. Thanks @hannesrudolph.
- Made `messages --sync` fail fast with an omit-`--sync` hint when a live `tail` process owns the sync lock, while plain `messages` reads continue without waiting. Thanks @jeanmonet.

## 0.9.1 - 2026-05-18

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ Notes:
- `--days` is shorthand for "since now minus N days"
- `--last` returns the newest `N` matching messages, then prints them oldest-to-newest
- `--all` removes the safety limit; default is `200`
- `--sync` runs a blocking pre-query sync for the matching channel or guild scope before reading the local DB
- `--sync` runs a blocking pre-query sync for the matching channel or guild scope before reading the local DB; omit it while `tail` is already maintaining live freshness
- rows with no displayable/searchable content are skipped by default; `--include-empty` opts back in
- at least one filter is required
- `--dm` is shorthand for `--guild @me`, so DM searches and message slices do not need raw SQL
Expand Down
3 changes: 2 additions & 1 deletion docs/commands/messages.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,13 @@ discrawl --json messages --channel maintainers --days 3
- `--last <n>` - return the newest `N` matching messages, then print oldest-to-newest
- `--limit <n>` - safety limit (default 200; `--all` removes it)
- `--all` - removes the safety limit
- `--sync` - blocking pre-query sync for the matching channel or guild scope
- `--sync` - blocking pre-query sync for the matching channel or guild scope; omit while `tail` is already maintaining live freshness
- `--include-empty` - include rows with no displayable/searchable content

## Notes

- at least one filter is required
- if `tail` is already running, plain `messages` reads the local archive without waiting; `messages --sync` fails fast instead of waiting behind the tail lock
- `--dm` skips Git snapshot auto-update because DMs are never imported from the shared mirror
- use either `--last` for the newest matching rows or `--all` for an uncapped oldest-to-newest slice

Expand Down
6 changes: 6 additions & 0 deletions internal/cli/admin_commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,12 @@ func (r *runtime) runTail(args []string) error {
}
ctx, stop := signal.NotifyContext(r.ctx, os.Interrupt, syscall.SIGTERM)
defer stop()
if configurable, ok := r.syncer.(tailReadyConfigurer); ok {
configurable.SetTailReadyCallback(func(context.Context) error {
return r.activateTailSyncLock()
})
defer configurable.SetTailReadyCallback(nil)
}
return r.syncer.RunTail(ctx, r.resolveSyncGuilds(*guildFlag, *guildsFlag), *repairEvery)
}

Expand Down
73 changes: 45 additions & 28 deletions internal/cli/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,25 +145,28 @@ func parseKongArgs(target any, args []string, name string, stdout, stderr io.Wri
}

type runtime struct {
ctx context.Context
configPath string
cfg config.Config
stdout io.Writer
stderr io.Writer
json bool
plain bool
logger *slog.Logger
store *store.Store
client discordClient
syncer syncService
dbLockHeld bool
lockStarted time.Time
openStore func(context.Context, string) (*store.Store, error)
newDiscord func(config.Config) (discordClient, error)
newRemote func(config.Config) (remoteArchiveClient, error)
newSyncer func(syncer.Client, *store.Store, *slog.Logger) syncService
newEmbed func(config.EmbeddingsConfig) (embed.Provider, error)
now func() time.Time
ctx context.Context
configPath string
cfg config.Config
stdout io.Writer
stderr io.Writer
json bool
plain bool
logger *slog.Logger
store *store.Store
client discordClient
syncer syncService
dbLockHeld bool
lockStarted time.Time
lockOperation string
lockToken string
lockTokenFree func() error
openStore func(context.Context, string) (*store.Store, error)
newDiscord func(config.Config) (discordClient, error)
newRemote func(config.Config) (remoteArchiveClient, error)
newSyncer func(syncer.Client, *store.Store, *slog.Logger) syncService
newEmbed func(config.EmbeddingsConfig) (embed.Provider, error)
now func() time.Time
}

func crawlkitEmbeddingConfig(cfg config.EmbeddingsConfig) embed.Config {
Expand All @@ -190,6 +193,10 @@ type syncService interface {
RunTail(context.Context, []string, time.Duration) error
}

type tailReadyConfigurer interface {
SetTailReadyCallback(func(context.Context) error)
}

type attachmentTextConfigurer interface {
SetAttachmentTextEnabled(bool)
}
Expand All @@ -209,7 +216,7 @@ func (r *runtime) dispatch(rest []string) error {
}
return r.withLocalStoreUpdateLocked(updateMode, true, func() error { return r.runSync(rest[1:]) })
case "tail":
return r.withServicesLocked(true, func() error { return r.runTail(rest[1:]) })
return r.withServicesLockedOperation(true, "tail-starting", func() error { return r.runTail(rest[1:]) })
case "wiretap":
return r.withLocalStoreLocked(false, func() error { return r.runWiretap(rest[1:]) })
case "tap", "cache-import":
Expand All @@ -235,10 +242,10 @@ func (r *runtime) dispatch(rest []string) error {
if r.configuredForCloudReadOnly() {
return r.withConfig(func() error { return r.runMessages(rest[1:]) })
}
if hasBoolFlag(rest[1:], "--sync") && !hasBoolFlag(rest[1:], "--dm") {
return r.withServicesAutoLocked(true, true, true, func() error { return r.runMessages(rest[1:]) })
if boolFlagEnabled(rest[1:], "--sync") && !boolFlagEnabled(rest[1:], "--dm") {
return r.withMessagesSyncServices(func() error { return r.runMessages(rest[1:]) })
}
autoShareUpdate := !hasBoolFlag(rest[1:], "--dm")
autoShareUpdate := !boolFlagEnabled(rest[1:], "--dm")
return r.withLocalStoreRead(autoShareUpdate, func() error { return r.runMessages(rest[1:]) })
case "digest":
return r.withLocalStoreRead(true, func() error { return r.runDigest(rest[1:]) })
Expand Down Expand Up @@ -323,8 +330,8 @@ func (r *runtime) withServices(withDiscord bool, fn func() error) error {
return r.withServicesAuto(withDiscord, !withDiscord, fn)
}

func (r *runtime) withServicesLocked(withDiscord bool, fn func() error) error {
return r.withServicesAutoLocked(withDiscord, !withDiscord, true, fn)
func (r *runtime) withServicesLockedOperation(withDiscord bool, operation string, fn func() error) error {
return r.withServicesUpdateLockedOperation(withDiscord, boolShareUpdateMode(!withDiscord), true, operation, fn)
}

func (r *runtime) withLocalStoreLocked(autoShareUpdate bool, fn func() error) error {
Expand Down Expand Up @@ -541,10 +548,14 @@ func (r *runtime) withServicesAuto(withDiscord, autoShareUpdate bool, fn func()
}

func (r *runtime) withServicesAutoLocked(withDiscord, autoShareUpdate, lockDB bool, fn func() error) error {
return r.withServicesUpdateLocked(withDiscord, boolShareUpdateMode(autoShareUpdate), lockDB, fn)
return r.withServicesUpdateLockedOperation(withDiscord, boolShareUpdateMode(autoShareUpdate), lockDB, "writer", fn)
}

func (r *runtime) withMessagesSyncServices(fn func() error) error {
return r.withServicesUpdateLockedOperation(true, shareUpdateConfigured, true, "messages-sync", fn)
}

func (r *runtime) withServicesUpdateLocked(withDiscord bool, updateMode shareUpdateMode, lockDB bool, fn func() error) error {
func (r *runtime) withServicesUpdateLockedOperation(withDiscord bool, updateMode shareUpdateMode, lockDB bool, operation string, fn func() error) error {
cfg, err := config.Load(r.configPath)
if err != nil {
return configErr(err)
Expand All @@ -558,7 +569,13 @@ func (r *runtime) withServicesUpdateLocked(withDiscord bool, updateMode shareUpd
}
r.cfg = cfg
if lockDB {
return r.withSyncLock(func() error {
lockFn := r.withSyncLockOperation
if operation == "messages-sync" {
lockFn = func(_ string, fn func() error) error {
return r.withMessagesSyncLock(fn)
}
}
return lockFn(operation, func() error {
return r.openServices(dbPath, withDiscord, updateMode, fn)
})
}
Expand Down
Loading