Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,18 @@ jobs:
GOOS: linux
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: '1'
MSGVAULT_OAUTH_CLIENT_ID: ${{ secrets.MSGVAULT_OAUTH_CLIENT_ID }}
MSGVAULT_OAUTH_CLIENT_SECRET: ${{ secrets.MSGVAULT_OAUTH_CLIENT_SECRET }}
run: |
if [ -z "$MSGVAULT_OAUTH_CLIENT_ID" ] || [ -z "$MSGVAULT_OAUTH_CLIENT_SECRET" ]; then
echo "FATAL: MSGVAULT_OAUTH_CLIENT_ID and MSGVAULT_OAUTH_CLIENT_SECRET repository secrets must be set" >&2
exit 1
fi
export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH"
VERSION=${GITHUB_REF#refs/tags/v}

mkdir -p dist
LDFLAGS="-s -w -X github.com/wesm/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X github.com/wesm/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X github.com/wesm/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -extldflags '-lstdc++ -lm'"
LDFLAGS="-s -w -X github.com/wesm/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X github.com/wesm/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X github.com/wesm/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -X github.com/wesm/msgvault/internal/oauth.oauthClientID=${MSGVAULT_OAUTH_CLIENT_ID} -X github.com/wesm/msgvault/internal/oauth.oauthClientSecret=${MSGVAULT_OAUTH_CLIENT_SECRET} -extldflags '-lstdc++ -lm'"
go build -tags "fts5 sqlite_vec" -trimpath -buildvcs=false -ldflags="$LDFLAGS" -o dist/msgvault ./cmd/msgvault

echo "--- Binary info ---"
Expand Down Expand Up @@ -106,11 +112,17 @@ jobs:
GOOS: darwin
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1
MSGVAULT_OAUTH_CLIENT_ID: ${{ secrets.MSGVAULT_OAUTH_CLIENT_ID }}
MSGVAULT_OAUTH_CLIENT_SECRET: ${{ secrets.MSGVAULT_OAUTH_CLIENT_SECRET }}
run: |
if [ -z "$MSGVAULT_OAUTH_CLIENT_ID" ] || [ -z "$MSGVAULT_OAUTH_CLIENT_SECRET" ]; then
echo "FATAL: MSGVAULT_OAUTH_CLIENT_ID and MSGVAULT_OAUTH_CLIENT_SECRET repository secrets must be set" >&2
exit 1
fi
VERSION=${GITHUB_REF#refs/tags/v}

mkdir -p dist
LDFLAGS="-s -w -X github.com/wesm/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X github.com/wesm/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X github.com/wesm/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
LDFLAGS="-s -w -X github.com/wesm/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X github.com/wesm/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X github.com/wesm/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -X github.com/wesm/msgvault/internal/oauth.oauthClientID=${MSGVAULT_OAUTH_CLIENT_ID} -X github.com/wesm/msgvault/internal/oauth.oauthClientSecret=${MSGVAULT_OAUTH_CLIENT_SECRET}"
go build -tags "fts5 sqlite_vec" -trimpath -ldflags="$LDFLAGS" -o dist/msgvault ./cmd/msgvault

echo "--- Binary info ---"
Expand Down Expand Up @@ -172,11 +184,17 @@ jobs:
CGO_ENABLED: '1'
CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline"
CGO_LDFLAGS: "-Wl,--allow-multiple-definition"
MSGVAULT_OAUTH_CLIENT_ID: ${{ secrets.MSGVAULT_OAUTH_CLIENT_ID }}
MSGVAULT_OAUTH_CLIENT_SECRET: ${{ secrets.MSGVAULT_OAUTH_CLIENT_SECRET }}
run: |
if [ -z "$MSGVAULT_OAUTH_CLIENT_ID" ] || [ -z "$MSGVAULT_OAUTH_CLIENT_SECRET" ]; then
echo "FATAL: MSGVAULT_OAUTH_CLIENT_ID and MSGVAULT_OAUTH_CLIENT_SECRET repository secrets must be set" >&2
exit 1
fi
VERSION="${GITHUB_REF#refs/tags/v}"

mkdir -p dist
LDFLAGS="-s -w -X github.com/wesm/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X github.com/wesm/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X github.com/wesm/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
LDFLAGS="-s -w -X github.com/wesm/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X github.com/wesm/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X github.com/wesm/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -X github.com/wesm/msgvault/internal/oauth.oauthClientID=${MSGVAULT_OAUTH_CLIENT_ID} -X github.com/wesm/msgvault/internal/oauth.oauthClientSecret=${MSGVAULT_OAUTH_CLIENT_SECRET}"
go build -tags "fts5 sqlite_vec" -trimpath -ldflags="$LDFLAGS" -o dist/msgvault.exe ./cmd/msgvault

# Smoke test
Expand Down
7 changes: 7 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ LDFLAGS := -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=$(VERSION) \
-X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=$(COMMIT) \
-X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(BUILD_DATE)

# Only inject embedded OAuth credentials when both env vars are set;
# otherwise leave the compiled-in defaults from internal/oauth/embedded.go.
ifneq ($(and $(MSGVAULT_OAUTH_CLIENT_ID),$(MSGVAULT_OAUTH_CLIENT_SECRET)),)
LDFLAGS += -X go.kenn.io/msgvault/internal/oauth.oauthClientID=$(MSGVAULT_OAUTH_CLIENT_ID) \
-X go.kenn.io/msgvault/internal/oauth.oauthClientSecret=$(MSGVAULT_OAUTH_CLIENT_SECRET)
endif

LDFLAGS_RELEASE := $(LDFLAGS) -s -w

# Default build tags applied to every go build/test/bench invocation.
Expand Down
41 changes: 31 additions & 10 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,16 +64,22 @@ conda install -c conda-forge msgvault

## Quick Start

> **Prerequisites:** You need a Google Cloud OAuth credential before adding an account.
> Follow the **[OAuth Setup Guide](https://msgvault.io/guides/oauth-setup/)** to create one (~5 minutes).

```bash
```sh
# Initialize the database
msgvault init-db
msgvault add-account you@gmail.com # opens browser for OAuth
msgvault sync-full you@gmail.com --limit 100

# Add a Gmail account — opens your browser for consent
msgvault add-account you@gmail.com

# Sync mail
msgvault sync-full you@gmail.com

# Browse the archive
msgvault tui
```

No Google Cloud Console setup required: msgvault ships with a verified OAuth client.

## Commands

| Command | Description |
Expand Down Expand Up @@ -129,14 +135,29 @@ All data lives in `~/.msgvault/` by default (override with `MSGVAULT_HOME`).

```toml
# ~/.msgvault/config.toml
[oauth]
client_secrets = "/path/to/client_secret.json"

[sync]
rate_limit_qps = 5
```

See the [Configuration Guide](https://msgvault.io/configuration/) for all options.
See the [Configuration Guide](https://msgvault.io/configuration/) for all options. To override the embedded OAuth client, see [Advanced: bring your own OAuth client](#advanced-bring-your-own-oauth-client) below.

### Advanced: bring your own OAuth client

The default flow uses msgvault's centralized verified OAuth client. You only need your own Cloud project if:

- Your Workspace organization prohibits authorizing third-party OAuth apps
- You prefer your own Cloud project's third-party-access listing to show
- You need your own Gmail API quota for very large mailboxes
- You want a fallback before msgvault's centralized client finishes Google verification

Follow the [OAuth setup guide](https://msgvault.io/guides/oauth-setup/) to create a Desktop OAuth client, then add it to `~/.msgvault/config.toml`:

```toml
[oauth]
client_secrets = "/path/to/client_secret.json"
```

Use `--oauth-app NAME` for per-account named-app routing — see the OAuth setup guide for details.

### Multiple OAuth Apps (Google Workspace)

Expand Down
15 changes: 3 additions & 12 deletions cmd/msgvault/cmd/addaccount.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ Examples:
// Resolve which client secrets to use
resolvedApp := oauthAppName
oauthAppExplicit := cmd.Flags().Changed("oauth-app")
var clientSecretsPath string

// Initialize database (in case it's new)
dbPath := cfg.DatabaseDSN()
Expand Down Expand Up @@ -165,21 +164,13 @@ Examples:
return nil
}

// Resolve client secrets path (standard OAuth flow)
clientSecretsPath, err = cfg.OAuth.ClientSecretsFor(resolvedApp)
// Build the OAuth manager. resolveOAuthManager handles named BYO,
// global BYO, and the embedded fallback automatically.
oauthMgr, err := resolveOAuthManager(cfg, resolvedApp, oauth.Scopes, logger)
if err != nil {
if !cfg.OAuth.HasAnyConfig() {
return errOAuthNotConfigured()
}
return err
}

// Create OAuth manager
oauthMgr, err := oauth.NewManager(clientSecretsPath, cfg.TokensDir(), logger)
if err != nil {
return wrapOAuthError(fmt.Errorf("create oauth manager: %w", err))
}

// If --force, delete existing token so we re-authorize
if forceReauth {
if oauthMgr.HasToken(email) {
Expand Down
61 changes: 61 additions & 0 deletions cmd/msgvault/cmd/addaccount_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/spf13/cobra"
"go.kenn.io/msgvault/internal/config"
"go.kenn.io/msgvault/internal/oauth"
"go.kenn.io/msgvault/internal/store"
)

Expand Down Expand Up @@ -1223,3 +1224,63 @@ func TestAddAccount_ForceServiceAccountReturnsActionableError(t *testing.T) {
t.Fatalf("error = %v, want service accounts do not use --force", err)
}
}

func TestAddAccount_ResolverBranches(t *testing.T) {
tests := []struct {
name string
appName string
setup func(cfg *config.Config, t *testing.T)
wantErr bool
errContains string
}{
{
name: "named BYO with client_secrets",
appName: "acme",
setup: func(cfg *config.Config, t *testing.T) {
path := writeStubClientSecrets(t, cfg.Data.DataDir, "acme.json")
cfg.OAuth.Apps = map[string]config.OAuthApp{"acme": {ClientSecrets: path}}
},
wantErr: false,
},
{
name: "named app without client_secrets",
appName: "missing",
setup: func(cfg *config.Config, t *testing.T) {},
wantErr: true,
errContains: "missing",
},
{
name: "global BYO",
appName: "",
setup: func(cfg *config.Config, t *testing.T) {
cfg.OAuth.ClientSecrets = writeStubClientSecrets(t, cfg.Data.DataDir, "default.json")
},
wantErr: false,
},
{
name: "no config falls through to embedded",
appName: "",
setup: func(cfg *config.Config, t *testing.T) {},
wantErr: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cfg := newTestConfig(t)
tc.setup(cfg, t)
_, err := resolveOAuthManager(cfg, tc.appName, oauth.Scopes, slog.Default())
if tc.wantErr {
if err == nil {
t.Fatal("expected error, got nil")
}
if tc.errContains != "" && !strings.Contains(err.Error(), tc.errContains) {
t.Errorf("error %q should contain %q", err.Error(), tc.errContains)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
})
}
}
39 changes: 9 additions & 30 deletions cmd/msgvault/cmd/deletions.go
Original file line number Diff line number Diff line change
Expand Up @@ -491,29 +491,19 @@ Examples:
// buildAPIClient uses standard scopes; deletion may need elevated ones.
// Service-account flows get scopes via the JWT assertion (no stored
// token), so the scope-escalation prompt only applies to browser OAuth.
var clientSecretsPath string
if src.SourceType == "gmail" {
if !cfg.OAuth.HasAnyConfig() {
return errOAuthNotConfigured()
}
appName := sourceOAuthApp(src)
isServiceAccount := cfg.OAuth.ServiceAccountKeyFor(appName) != ""

if !isServiceAccount {
clientSecretsPath, err = cfg.OAuth.ClientSecretsFor(appName)
if err != nil {
return err
}

needsBatchDelete := deletePermanent
if needsBatchDelete {
requiredScopes := oauth.ScopesDeletion
oauthMgr, err := oauth.NewManagerWithScopes(clientSecretsPath, cfg.TokensDir(), logger, requiredScopes)
oauthMgr, err := resolveOAuthManager(cfg, appName, oauth.ScopesDeletion, logger)
if err != nil {
return wrapOAuthError(fmt.Errorf("create oauth manager: %w", err))
return err
}
if !oauthMgr.HasScope(account, "https://mail.google.com/") && oauthMgr.HasScopeMetadata(account) {
if err := promptScopeEscalation(ctx, oauthMgr, account, needsBatchDelete, clientSecretsPath); err != nil {
if err := promptScopeEscalation(ctx, oauthMgr, account, needsBatchDelete, appName); err != nil {
if errors.Is(err, errUserCanceled) {
return nil
}
Expand All @@ -526,19 +516,11 @@ Examples:

// Build API client — reuses the same factory as sync.
getOAuthMgr := func(appName string) (*oauth.Manager, error) {
secretsPath := clientSecretsPath
if secretsPath == "" {
var err error
secretsPath, err = cfg.OAuth.ClientSecretsFor(appName)
if err != nil {
return nil, err
}
}
scopes := oauth.Scopes
if deletePermanent {
scopes = oauth.ScopesDeletion
}
return oauth.NewManagerWithScopes(secretsPath, cfg.TokensDir(), logger, scopes)
return resolveOAuthManager(cfg, appName, scopes, logger)
}
// For permanent deletion (not trash), service-account flows need the
// elevated mail.google.com scope; trash-only uses the standard set.
Expand Down Expand Up @@ -602,7 +584,7 @@ Examples:
if mgrErr != nil {
return mgrErr
}
if err := promptScopeEscalation(ctx, oauthMgr, account, !useTrash, clientSecretsPath); err != nil {
if err := promptScopeEscalation(ctx, oauthMgr, account, !useTrash, sourceOAuthApp(src)); err != nil {
if errors.Is(err, errUserCanceled) {
return nil
}
Expand Down Expand Up @@ -699,10 +681,7 @@ func (p *CLIDeletionProgress) OnProgress(processed, succeeded, failed int) {
}

func (p *CLIDeletionProgress) progressBar(pct float64, width int) string {
filled := int(pct / 100 * float64(width))
if filled > width {
filled = width
}
filled := min(int(pct/100*float64(width)), width)
bar := make([]byte, width)
for i := range bar {
if i < filled {
Expand Down Expand Up @@ -748,7 +727,7 @@ var errUserCanceled = errors.New("user canceled scope escalation")
// promptScopeEscalation prompts the user to re-authorize with elevated scopes.
// It deletes the old token, runs the OAuth browser flow, and returns nil on
// success. The caller should re-create the OAuth manager after this returns.
func promptScopeEscalation(ctx context.Context, oauthMgr *oauth.Manager, account string, batchDelete bool, clientSecretsPath string) error {
func promptScopeEscalation(ctx context.Context, oauthMgr *oauth.Manager, account string, batchDelete bool, appName string) error {
fmt.Println("\n" + strings.Repeat("=", 70))
fmt.Println("PERMISSION UPGRADE REQUIRED")
fmt.Println(strings.Repeat("=", 70))
Expand Down Expand Up @@ -798,9 +777,9 @@ func promptScopeEscalation(ctx context.Context, oauthMgr *oauth.Manager, account
fmt.Println("Starting OAuth flow...")
fmt.Println()

newMgr, err := oauth.NewManagerWithScopes(clientSecretsPath, cfg.TokensDir(), logger, requiredScopes)
newMgr, err := resolveOAuthManager(cfg, appName, requiredScopes, logger)
if err != nil {
return fmt.Errorf("create oauth manager: %w", err)
return err
}

if err := newMgr.Authorize(ctx, account); err != nil {
Expand Down
49 changes: 49 additions & 0 deletions cmd/msgvault/cmd/oauth_resolve.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package cmd

import (
"fmt"
"log/slog"

"go.kenn.io/msgvault/internal/config"
"go.kenn.io/msgvault/internal/oauth"
)

// resolveOAuthManager builds the *oauth.Manager appropriate for the
// account+config+scopes triple. Resolution order:
//
// 1. Named BYO: appName is non-empty and cfg.OAuth.Apps[appName] has
// client_secrets set — use that BYO OAuth client.
// 2. (If appName is non-empty but no client_secrets is registered for
// it) — return an error rather than falling through to embedded,
// because the user explicitly named a binding that doesn't exist.
// 3. Global BYO: appName is empty and cfg.OAuth.ClientSecrets is set —
// use the global BYO client.
// 4. Embedded: otherwise, use the centralized verified client. On the
// embedded path the manager is always built with oauth.ScopesEmbedded,
// ignoring the caller's per-call scope choice, because the embedded
// grant is broader than any per-call need.
//
// Callers handle service-account resolution themselves (via
// cfg.OAuth.ServiceAccountKeyFor(appName)) before calling this helper,
// because *oauth.Manager and the service-account manager have
// different interfaces.
func resolveOAuthManager(
cfg *config.Config,
appName string,
scopes []string,
logger *slog.Logger,
) (*oauth.Manager, error) {
if appName != "" {
app, ok := cfg.OAuth.Apps[appName]
if !ok || app.ClientSecrets == "" {
return nil, fmt.Errorf("OAuth app %q not configured: add [oauth.apps.%s] client_secrets to config.toml, or rebind the account with 'msgvault add-account <email>' (without --oauth-app) to use the embedded client", appName, appName)
}
return oauth.NewManagerWithScopes(app.ClientSecrets, cfg.TokensDir(), logger, scopes)
}

if cfg.OAuth.ClientSecrets != "" {
return oauth.NewManagerWithScopes(cfg.OAuth.ClientSecrets, cfg.TokensDir(), logger, scopes)
}

return oauth.NewEmbeddedManager(cfg.TokensDir(), logger, oauth.ScopesEmbedded)
}
Loading
Loading