code cleanup

Author: Thomas Stromberg

pkg/cache/cache.go

@@ -19,7 +19,7 @@ const errorTTL = 5 * 24 * time.Hour // Cache HTTP errors for 5 days
 var globalRateLimiter = newGlobalRateLimiter()
 
 func newGlobalRateLimiter() *DomainRateLimiter {
-	r := NewDomainRateLimiter(600 * time.Millisecond)
+	r := NewDomainRateLimiter(200 * time.Millisecond)
 	r.SetDomainDelay("www.linkedin.com", 1200*time.Millisecond)
 	return r
 }

pkg/cache/ratelimit.go

@@ -34,10 +34,11 @@ func (r *DomainRateLimiter) SetDomainDelay(domain string, delay time.Duration) {
 // Wait blocks until it's safe to make a request to the given URL's domain.
 // It ensures at least minDelay has passed since the last request to that domain.
 func (r *DomainRateLimiter) Wait(rawURL string) {
-	domain := extractDomain(rawURL)
-	if domain == "" {
+	u, err := url.Parse(rawURL)
+	if err != nil || u.Host == "" {
 		return
 	}
+	domain := u.Host
 
 	// Get or create per-domain mutex
 	muI, _ := r.mu.LoadOrStore(domain, &sync.Mutex{})
@@ -70,12 +71,3 @@ func (r *DomainRateLimiter) Wait(rawURL string) {
 	// Record this request
 	r.lastRequest.Store(domain, time.Now())
 }
-
-// extractDomain returns the host portion of a URL, or empty string on error.
-func extractDomain(rawURL string) string {
-	u, err := url.Parse(rawURL)
-	if err != nil {
-		return ""
-	}
-	return u.Host
-}

pkg/github/github.go

@@ -224,6 +224,16 @@ func (e *APIError) Error() string {
 }
 
 func (c *Client) fetchAPI(ctx context.Context, urlStr, username string) (*profile.Profile, error) {
+	// Try GraphQL first (gets social accounts), fall back to REST API
+	if c.token != "" {
+		prof, err := c.fetchGraphQL(ctx, urlStr, username)
+		if err == nil {
+			return prof, nil
+		}
+		c.logger.WarnContext(ctx, "GraphQL fetch failed, falling back to REST API", "error", err)
+	}
+
+	// REST API fallback
 	apiURL := "https://api.github.com/users/" + username
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, http.NoBody)
@@ -245,6 +255,161 @@ func (c *Client) fetchAPI(ctx context.Context, urlStr, username string) (*profil
 	return parseJSON(body, urlStr, username)
 }
 
+func (c *Client) fetchGraphQL(ctx context.Context, urlStr, username string) (*profile.Profile, error) {
+	query := `
+	query($login: String!) {
+		user(login: $login) {
+			name
+			login
+			location
+			bio
+			company
+			websiteUrl
+			twitterUsername
+			createdAt
+			updatedAt
+
+			socialAccounts(first: 10) {
+				nodes {
+					provider
+					url
+					displayName
+				}
+			}
+
+			followers {
+				totalCount
+			}
+			following {
+				totalCount
+			}
+
+			repositories(first: 1, ownerAffiliations: OWNER) {
+				totalCount
+			}
+		}
+	}
+	`
+
+	variables := map[string]string{"login": username}
+	reqBody := map[string]any{
+		"query":     query,
+		"variables": variables,
+	}
+
+	jsonData, err := json.Marshal(reqBody)
+	if err != nil {
+		return nil, fmt.Errorf("marshaling GraphQL request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://api.github.com/graphql", strings.NewReader(string(jsonData)))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+c.token)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", "sociopath/1.0")
+
+	body, err := c.doAPIRequest(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	return parseGraphQLResponse(body, urlStr, username)
+}
+
+func parseGraphQLResponse(data []byte, urlStr, _ string) (*profile.Profile, error) {
+	var response struct {
+		Errors []struct {
+			Message string `json:"message"`
+		} `json:"errors"`
+		Data struct {
+			User struct {
+				Name           string `json:"name"`
+				Login          string `json:"login"`
+				Location       string `json:"location"`
+				Bio            string `json:"bio"`
+				Company        string `json:"company"`
+				WebsiteURL     string `json:"websiteUrl"`
+				TwitterUser    string `json:"twitterUsername"`
+				SocialAccounts struct {
+					Nodes []struct {
+						URL         string `json:"url"`
+						Provider    string `json:"provider"`
+						DisplayName string `json:"displayName"`
+					} `json:"nodes"`
+				} `json:"socialAccounts"`
+				Followers    struct{ TotalCount int } `json:"followers"`
+				Following    struct{ TotalCount int } `json:"following"`
+				Repositories struct{ TotalCount int } `json:"repositories"`
+			} `json:"user"`
+		} `json:"data"`
+	}
+
+	if err := json.Unmarshal(data, &response); err != nil {
+		return nil, fmt.Errorf("parsing GraphQL response: %w", err)
+	}
+
+	if len(response.Errors) > 0 {
+		return nil, fmt.Errorf("GraphQL error: %s", response.Errors[0].Message)
+	}
+
+	user := response.Data.User
+	prof := &profile.Profile{
+		Platform:      platform,
+		URL:           urlStr,
+		Authenticated: true,
+		Username:      user.Login,
+		Name:          user.Name,
+		Bio:           user.Bio,
+		Location:      user.Location,
+		Fields:        make(map[string]string),
+	}
+
+	// Add website
+	if user.WebsiteURL != "" {
+		website := user.WebsiteURL
+		if !strings.HasPrefix(website, "http") {
+			website = "https://" + website
+		}
+		prof.Website = website
+		prof.Fields["website"] = website
+	}
+
+	// Add company
+	if user.Company != "" {
+		company := strings.TrimPrefix(user.Company, "@")
+		prof.Fields["company"] = company
+	}
+
+	// Add stats
+	if user.Repositories.TotalCount > 0 {
+		prof.Fields["public_repos"] = strconv.Itoa(user.Repositories.TotalCount)
+	}
+	if user.Followers.TotalCount > 0 {
+		prof.Fields["followers"] = strconv.Itoa(user.Followers.TotalCount)
+	}
+	if user.Following.TotalCount > 0 {
+		prof.Fields["following"] = strconv.Itoa(user.Following.TotalCount)
+	}
+
+	// Add Twitter from GraphQL
+	if user.TwitterUser != "" {
+		twitterURL := "https://twitter.com/" + user.TwitterUser
+		prof.Fields["twitter"] = twitterURL
+		prof.SocialLinks = append(prof.SocialLinks, twitterURL)
+	}
+
+	// Add social accounts from GraphQL - this is the key improvement!
+	for _, social := range user.SocialAccounts.Nodes {
+		if social.URL != "" {
+			prof.SocialLinks = append(prof.SocialLinks, social.URL)
+		}
+	}
+
+	return prof, nil
+}
+
 func (c *Client) doAPIRequest(ctx context.Context, req *http.Request) ([]byte, error) {
 	// Check cache first
 	cacheKey := req.URL.String()
@@ -272,8 +437,10 @@ func (c *Client) doAPIRequest(ctx context.Context, req *http.Request) ([]byte, e
 	defer func() { _ = resp.Body.Close() }() //nolint:errcheck // error ignored intentionally
 
 	// Parse rate limit headers (GitHub uses non-canonical casing, parse errors default to 0)
-	rateLimitRemain, _ := strconv.Atoi(resp.Header.Get("X-RateLimit-Remaining"))        //nolint:errcheck,canonicalheader // ok
-	rateLimitReset, _ := strconv.ParseInt(resp.Header.Get("X-RateLimit-Reset"), 10, 64) //nolint:errcheck,canonicalheader // ok
+	//nolint:errcheck,canonicalheader // GitHub uses non-canonical header casing
+	rateLimitRemain, _ := strconv.Atoi(resp.Header.Get("X-RateLimit-Remaining"))
+	//nolint:errcheck,canonicalheader // GitHub uses non-canonical header casing
+	rateLimitReset, _ := strconv.ParseInt(resp.Header.Get("X-RateLimit-Reset"), 10, 64)
 	resetTime := time.Unix(rateLimitReset, 0)
 
 	if resp.StatusCode != http.StatusOK {

pkg/sociopath/sociopath.go

@@ -71,9 +71,10 @@ var (
 type Option func(*config)
 
 type config struct {
-	cookies        map[string]string
 	cache          cache.HTTPCache
+	cookies        map[string]string
 	logger         *slog.Logger
+	githubToken    string
 	browserCookies bool
 }
 
@@ -97,6 +98,11 @@ func WithLogger(logger *slog.Logger) Option {
 	return func(c *config) { c.logger = logger }
 }
 
+// WithGitHubToken sets the GitHub API token for authenticated requests.
+func WithGitHubToken(token string) Option {
+	return func(c *config) { c.githubToken = token }
+}
+
 // Fetch retrieves a profile from the given URL.
 // The platform is automatically detected from the URL.
 func Fetch(ctx context.Context, url string, opts ...Option) (*profile.Profile, error) {
@@ -374,6 +380,9 @@ func fetchGitHub(ctx context.Context, url string, cfg *config) (*profile.Profile
 	if cfg.logger != nil {
 		opts = append(opts, github.WithLogger(cfg.logger))
 	}
+	if cfg.githubToken != "" {
+		opts = append(opts, github.WithToken(cfg.githubToken))
+	}
 
 	client, err := github.New(ctx, opts...)
 	if err != nil {