Add CreatedAt timestamp to each source

Author: Thomas Stromberg

pkg/bluesky/bluesky.go

@@ -101,8 +101,8 @@ func (c *Client) Fetch(ctx context.Context, urlStr string) (*profile.Profile, er
 	// Fetch recent posts
 	posts, lastActive := c.fetchPosts(ctx, handle, 50)
 	p.Posts = posts
-	if lastActive != "" {
-		p.LastActive = lastActive
+	if lastActive != "" && lastActive > p.UpdatedAt {
+		p.UpdatedAt = lastActive
 	}
 
 	return p, nil
@@ -131,7 +131,7 @@ func parseAPIResponse(data []byte, urlStr, handle string) (*profile.Profile, err
 	}
 
 	if resp.CreatedAt != "" {
-		p.Fields["joined"] = resp.CreatedAt
+		p.CreatedAt = resp.CreatedAt
 	}
 
 	// Extract hashtags from bio

pkg/codeberg/codeberg.go

@@ -189,7 +189,7 @@ func parseHTML(data []byte, urlStr, username string) *profile.Profile {
 	// Pattern: Joined on 2023-04-06
 	joinedPattern := regexp.MustCompile(`Joined\s+on\s+(\d{4}-\d{2}-\d{2})`)
 	if m := joinedPattern.FindStringSubmatch(content); len(m) > 1 {
-		prof.Fields["joined"] = m[1]
+		prof.CreatedAt = m[1]
 	}
 
 	// Extract follower/following counts

pkg/codeberg/codeberg_test.go

@@ -169,12 +169,12 @@ func TestFetch_InvalidUsername(t *testing.T) {
 
 func TestParseHTML(t *testing.T) {
 	tests := []struct {
-		name         string
-		html         string
-		username     string
-		wantName     string
-		wantPronouns string
-		wantJoined   string
+		name          string
+		html          string
+		username      string
+		wantName      string
+		wantPronouns  string
+		wantCreatedAt string
 	}{
 		{
 			name: "full profile with pronouns",
@@ -185,10 +185,10 @@ func TestParseHTML(t *testing.T) {
 				<div>0 followers · 0 following</div>
 				<div>Joined on 2023-04-06</div>
 			</body></html>`,
-			username:     "johwhj",
-			wantName:     "Woohyun Joh",
-			wantPronouns: "he/him",
-			wantJoined:   "2023-04-06",
+			username:      "johwhj",
+			wantName:      "Woohyun Joh",
+			wantPronouns:  "he/him",
+			wantCreatedAt: "2023-04-06",
 		},
 		{
 			name: "profile without pronouns",
@@ -198,9 +198,9 @@ func TestParseHTML(t *testing.T) {
 				<span class="username">stephen-fox</span>
 				<div>Joined on 2025-02-15</div>
 			</body></html>`,
-			username:   "stephen-fox",
-			wantName:   "stephen-fox",
-			wantJoined: "2025-02-15",
+			username:      "stephen-fox",
+			wantName:      "stephen-fox",
+			wantCreatedAt: "2025-02-15",
 		},
 		{
 			name: "organization profile",
@@ -224,8 +224,8 @@ func TestParseHTML(t *testing.T) {
 			if tt.wantPronouns != "" && profile.Fields["pronouns"] != tt.wantPronouns {
 				t.Errorf("Pronouns = %q, want %q", profile.Fields["pronouns"], tt.wantPronouns)
 			}
-			if tt.wantJoined != "" && profile.Fields["joined"] != tt.wantJoined {
-				t.Errorf("Joined = %q, want %q", profile.Fields["joined"], tt.wantJoined)
+			if tt.wantCreatedAt != "" && profile.CreatedAt != tt.wantCreatedAt {
+				t.Errorf("Joined = %q, want %q", profile.CreatedAt, tt.wantCreatedAt)
 			}
 		})
 	}

pkg/devto/devto.go

@@ -97,8 +97,8 @@ func (c *Client) Fetch(ctx context.Context, urlStr string) (*profile.Profile, er
 	// Fetch recent articles via API
 	posts, lastActive := c.fetchArticles(ctx, username, 50)
 	p.Posts = posts
-	if lastActive != "" {
-		p.LastActive = lastActive
+	if lastActive != "" && lastActive > p.UpdatedAt {
+		p.UpdatedAt = lastActive
 	}
 
 	return p, nil
@@ -145,8 +145,7 @@ func parseHTML(data []byte, urlStr, username string) *profile.Profile {
 	// Extract joined date
 	joinedPattern := regexp.MustCompile(`<time\s+datetime="([^"]+)"[^>]*>([^<]+)</time>`)
 	if m := joinedPattern.FindStringSubmatch(content); len(m) > 2 {
-		p.Fields["joined"] = strings.TrimSpace(m[2])
-		p.Fields["joined_datetime"] = m[1]
+		p.CreatedAt = m[1] // ISO datetime format
 	}
 
 	// Extract work/employment - look for <p>Work</p> followed by value

pkg/generic/generic.go

@@ -146,9 +146,9 @@ func parseHTML(data []byte, urlStr string) *profile.Profile {
 		p.Posts = posts
 		p.Platform = "blog"
 		if lastActive != "" {
-			p.LastActive = lastActive
+			p.UpdatedAt = lastActive
 		} else if len(posts) > 0 && posts[0].URL != "" {
-			p.LastActive = extractDateFromURL(posts[0].URL)
+			p.UpdatedAt = extractDateFromURL(posts[0].URL)
 		}
 	}
 

pkg/github/github.go

@@ -335,6 +335,8 @@ func parseGraphQLResponse(data []byte, urlStr, _ string) (*profile.Profile, erro
 				Company        string `json:"company"`
 				WebsiteURL     string `json:"websiteUrl"`
 				TwitterUser    string `json:"twitterUsername"`
+				CreatedAt      string `json:"createdAt"`
+				UpdatedAt      string `json:"updatedAt"`
 				SocialAccounts struct {
 					Nodes []struct {
 						URL         string `json:"url"`
@@ -410,6 +412,14 @@ func parseGraphQLResponse(data []byte, urlStr, _ string) (*profile.Profile, erro
 		}
 	}
 
+	// Add account timestamps
+	if user.CreatedAt != "" {
+		prof.CreatedAt = user.CreatedAt
+	}
+	if user.UpdatedAt != "" {
+		prof.UpdatedAt = user.UpdatedAt
+	}
+
 	return prof, nil
 }
 
@@ -450,10 +460,10 @@ func (c *Client) doAPIRequest(ctx context.Context, req *http.Request) ([]byte, e
 	}
 	defer func() { _ = resp.Body.Close() }() //nolint:errcheck // error ignored intentionally
 
-	// Parse rate limit headers (GitHub uses non-canonical casing, parse errors default to 0)
-	//nolint:errcheck,canonicalheader // GitHub uses non-canonical header casing
+	// Parse rate limit headers (parse errors default to 0)
+	//nolint:errcheck,canonicalheader // GitHub uses non-canonical header casing, parse errors acceptable
 	rateLimitRemain, _ := strconv.Atoi(resp.Header.Get("X-RateLimit-Remaining"))
-	//nolint:errcheck,canonicalheader // GitHub uses non-canonical header casing
+	//nolint:errcheck,canonicalheader // GitHub uses non-canonical header casing, parse errors acceptable
 	rateLimitReset, _ := strconv.ParseInt(resp.Header.Get("X-RateLimit-Reset"), 10, 64)
 	resetTime := time.Unix(rateLimitReset, 0)
 
@@ -638,6 +648,8 @@ func parseJSON(data []byte, urlStr, _ string) (*profile.Profile, error) {
 		AvatarURL   string `json:"avatar_url"`
 		HTMLURL     string `json:"html_url"`
 		Type        string `json:"type"`
+		CreatedAt   string `json:"created_at"`
+		UpdatedAt   string `json:"updated_at"`
 	}
 
 	if err := json.Unmarshal(data, &ghUser); err != nil {
@@ -722,6 +734,14 @@ func parseJSON(data []byte, urlStr, _ string) (*profile.Profile, error) {
 		prof.Fields["type"] = ghUser.Type
 	}
 
+	// Add account timestamps
+	if ghUser.CreatedAt != "" {
+		prof.CreatedAt = ghUser.CreatedAt
+	}
+	if ghUser.UpdatedAt != "" {
+		prof.UpdatedAt = ghUser.UpdatedAt
+	}
+
 	return prof, nil
 }
 

pkg/mastodon/mastodon.go

@@ -155,8 +155,8 @@ func (c *Client) fetchViaAPI(ctx context.Context, host, username string) (*profi
 	if accountID != "" {
 		posts, lastActive := c.fetchStatuses(ctx, host, accountID, 50)
 		p.Posts = posts
-		if lastActive != "" {
-			p.LastActive = lastActive
+		if lastActive != "" && lastActive > p.UpdatedAt {
+			p.UpdatedAt = lastActive
 		}
 	}
 
@@ -210,6 +210,11 @@ func (*Client) parseAPIResponse(data []byte) (*profile.Profile, string, error) {
 	// Filter out same-server Mastodon links
 	p.SocialLinks = filterSameServerLinks(p.SocialLinks, p.URL)
 
+	// Add account creation date
+	if acc.CreatedAt != "" {
+		p.CreatedAt = acc.CreatedAt
+	}
+
 	return p, acc.ID, nil
 }
 

pkg/profile/profile.go

@@ -47,18 +47,17 @@ type Profile struct {
 	Error         string `json:",omitempty"` // Error message if fetch failed (e.g., "login required")
 
 	// Core profile data
-	Username string `json:",omitempty"` // Handle/username (without @ prefix)
-	Name     string `json:",omitempty"` // Display name
-	Bio      string `json:",omitempty"` // Profile bio/description
-	Location string `json:",omitempty"` // Geographic location
-	Website  string `json:",omitempty"` // Personal website URL
+	Username  string `json:",omitempty"` // Handle/username (without @ prefix)
+	Name      string `json:",omitempty"` // Display name
+	Bio       string `json:",omitempty"` // Profile bio/description
+	Location  string `json:",omitempty"` // Geographic location
+	Website   string `json:",omitempty"` // Personal website URL
+	CreatedAt string `json:",omitempty"` // Account creation date (ISO timestamp)
+	UpdatedAt string `json:",omitempty"` // Most recent activity or profile update (ISO timestamp)
 
 	// Platform-specific fields
 	Fields map[string]string `json:",omitempty"` // Additional platform-specific data (headline, employer, etc.)
 
-	// Activity timestamp
-	LastActive string `json:",omitempty"` // ISO timestamp of last known activity (post, comment, etc.)
-
 	// For further crawling
 	SocialLinks []string `json:",omitempty"` // Other social media URLs detected on the profile
 

pkg/reddit/reddit.go

@@ -126,7 +126,7 @@ func parseProfile(html, url, username string) (*profile.Profile, error) {
 	// Extract cake day (account creation date)
 	cakeDayPattern := regexp.MustCompile(`(?i)redditor since.*?(\d{4})`)
 	if matches := cakeDayPattern.FindStringSubmatch(html); len(matches) > 1 {
-		prof.Fields["joined_year"] = matches[1]
+		prof.CreatedAt = matches[1] // Year only
 	}
 
 	// Extract posts and comments with subreddit context