fetch_scholar.py

#!/usr/bin/env python3
import time
import sys
import csv
import requests
import os
from datetime import datetime, timezone
from scholarly import scholarly
from dotenv import load_dotenv
from db import SessionLocal, init_db
from models import Author, Publication

# --- Load environment variables (make sure your .env has SCOPUS_API_KEY) ---
load_dotenv()
SCOPUS_API_KEY = os.getenv("SCOPUS_API_KEY")

RATE_SLEEP = 0.001  # seconds between publication requests
LOG_FILE = "progress.log"


def log_message(message: str):
    """Append message to console and to progress.log"""
    timestamp = datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
    line = f"{timestamp} {message}"
    print(line)
    with open(LOG_FILE, "a", encoding="utf-8") as f:
        f.write(line + "\n")


# --- Check Scopus verification ---
def check_scopus_verification(title, doi=None):
    """Check if publication is indexed in Scopus (Elsevier API)."""
    if not SCOPUS_API_KEY:
        return False  # skip if API key not provided

    headers = {"X-ELS-APIKey": SCOPUS_API_KEY, "Accept": "application/json"}
    base_url = "https://api.elsevier.com/content/search/scopus"
    params = {"query": f"DOI({doi})"} if doi else {"query": f'TITLE("{title}")'}

    try:
        r = requests.get(base_url, headers=headers, params=params, timeout=15)
        if r.status_code == 200:
            data = r.json()
            total_results = int(data.get("search-results", {}).get("opensearch:totalResults", 0))
            return total_results > 0
        else:
            log_message(f"[!] Scopus API error {r.status_code} for: {title}")
            return False
    except Exception as e:
        log_message(f"[!] Scopus check failed: {e}")
        return False


def extract_year(bib):
    """Extract publication year from bib dict or alternative keys."""
    for key in ("year", "pub_year", "publication_year", "date"):
        if key in bib and bib[key]:
            try:
                return int(bib[key])
            except ValueError:
                continue
    return None


def fetch_author_by_name_or_id(query):
    """Detect if query is a name or Scholar ID, fetch author accordingly."""
    try:
        if len(query) >= 10 and query.isalnum():
            log_message(f"[*] Fetching author by Scholar ID: {query}")
            author_summary = scholarly.search_author_id(query)
            author_filled = scholarly.fill(author_summary, sections=["basics", "indices", "publications"])
        else:
            log_message(f"[*] Searching author by name: {query}")
            results = scholarly.search_author(query)
            author = next(results, None)
            if not author:
                log_message(f"[!] No author found for query: {query}")
                return None
            author_filled = scholarly.fill(author, sections=["basics", "indices", "publications"])

        # Log author stats
        name = author_filled.get("name")
        hindex = author_filled.get("hindex", 0)
        i10 = author_filled.get("i10index", 0)
        citedby = author_filled.get("citedby", 0)

        log_message(f"✅ Found author: {name}")
        log_message(f"📊 h-index: {hindex} | i10-index: {i10} | Total Citations: {citedby}")
        return author_filled

    except Exception as e:
        log_message(f"[!] Error fetching author: {e}")
        return None


def store_author_and_pubs(author_filled):
    """Store author and publication data in MySQL + CSV, with progress log."""
    session = SessionLocal()
    try:
        name = author_filled.get("name")
        scholar_id = author_filled.get("scholar_id") or author_filled.get("id") or author_filled.get("user_id")
        citedby = int(author_filled.get("citedby", 0))
        hindex = int(author_filled.get("hindex", 0))
        i10 = int(author_filled.get("i10index", 0)) if author_filled.get("i10index") else 0

        # Upsert author
        author_row = session.query(Author).filter(
            (Author.scholar_id == scholar_id) | (Author.name == name)
        ).first()

        if not author_row:
            log_message(f"[*] Adding new author: {name}")
            author_row = Author(
                name=name,
                scholar_id=scholar_id,
                total_citations=citedby,
                h_index=hindex,
                i10_index=i10,
                last_updated=datetime.now(timezone.utc),
            )
            session.add(author_row)
            session.commit()
        else:
            log_message(f"[*] Updating existing author: {name}")
            author_row.total_citations = citedby
            author_row.h_index = hindex
            author_row.i10_index = i10
            author_row.last_updated = datetime.now(timezone.utc)
            session.commit()

        pubs = author_filled.get("publications", [])
        total_pubs = len(pubs)
        log_message(f"[*] Found {total_pubs} publications. Fetching details...\n")

        # --- Write header info + publications to CSV ---
        csv_file = f"{name.replace(' ', '_').lower()}_publications.csv"
        with open(csv_file, "w", newline="", encoding="utf-8") as csvfile:
            writer = csv.writer(csvfile)

            # 🧾 Write author metrics on top
            writer.writerow(["Author Name", name])
            writer.writerow(["h-index", hindex])
            writer.writerow(["i10-index", i10])
            writer.writerow(["Total Citations", citedby])
            writer.writerow([])  # empty line
            writer.writerow(["Title", "Year", "Citations", "Coauthors", "DOI", "URL", "Scopus Verified"])

            count = 0
            for pub in pubs:
                try:
                    pub_filled = scholarly.fill(pub)
                except Exception as e:
                    log_message(f"[!] Error filling publication: {e}")
                    continue

                bib = pub_filled.get("bib", {})
                title = bib.get("title", "").strip()
                year = extract_year(bib)
                citations = int(pub_filled.get("num_citations", 0))

                # --- Fetch and prioritize full paper URL ---
                bib_url = (
                    pub_filled.get("pub_url")
                    or bib.get("url")
                    or bib.get("eprint")
                    or pub_filled.get("source")
                    or None
                )

                doi = bib.get("doi") if "doi" in bib else None
                coauthors = ", ".join(bib.get("author", "").split(" and ")) if bib.get("author") else None

                doi_link = f"https://doi.org/{doi}" if doi else "No DOI"
                clickable_url = bib_url if bib_url else "No URL"

                # Check Scopus verification
                scopus_verified = check_scopus_verification(title, doi)
                status = "✅ Verified" if scopus_verified else "❌ Not Verified"

                log_message(
                    f"📄 {count+1}/{total_pubs}: {title[:90]} ({year or 'N/A'}) — Citations: {citations} — {status}"
                )

                # --- Database upsert ---
                existing = session.query(Publication).filter_by(author_id=author_row.id, title=title).first()
                if not existing:
                    new_pub = Publication(
                        author_id=author_row.id,
                        title=title,
                        year=year,
                        citations=citations,
                        bib_url=clickable_url,
                        doi=doi,
                        coauthors=coauthors,
                        scopus_verified=scopus_verified,
                    )
                    session.add(new_pub)
                else:
                    existing.citations = citations
                    existing.year = year
                    existing.bib_url = clickable_url
                    existing.doi = doi
                    existing.coauthors = coauthors
                    existing.scopus_verified = scopus_verified

                # --- Write to CSV ---
                writer.writerow([title, year, citations, coauthors, doi_link, clickable_url, status])

                count += 1
                if count % 10 == 0:
                    session.commit()
                    log_message(f"💾 Saved {count}/{total_pubs} so far...")

                time.sleep(RATE_SLEEP)

            session.commit()
            log_message(f"\n✅ Done. Total {count} publications saved for {name}.")
            log_message(f"📁 CSV saved as '{csv_file}' in your project directory.\n")

    except Exception as e:
        log_message(f"[!] Database error: {e}")
        session.rollback()
    finally:
        session.close()


def main():
    init_db()
    if len(sys.argv) < 2:
        print("Usage: python fetch_scholar.py 'Author Name or ScholarID'")
        sys.exit(1)

    query = sys.argv[1].strip()
    author_filled = fetch_author_by_name_or_id(query)
    if not author_filled:
        log_message("[!] No author found or error.")
        sys.exit(1)

    store_author_and_pubs(author_filled)


if __name__ == "__main__":
    main()