Merge pull request #5 from linuxserver/add-new-fields

Add new fields

README.md

@@ -35,6 +35,7 @@ docker run -d \
 -e DB_FILE=/config/api.db `#optional` \
 -e INVALIDATE_HOURS=24 `#optional` \
 -e PAT=token `#optional` \
+-e SCARF_TOKEN=token `#optional` \
 -e URL=http://localhost:8000 `#optional` \
 -p 8000:8000 \
 -v /path/to/lsio-api/config:/config \

readme-vars.yml

@@ -40,6 +40,7 @@ full_custom_readme: |
   -e DB_FILE=/config/api.db `#optional` \
   -e INVALIDATE_HOURS=24 `#optional` \
   -e PAT=token `#optional` \
+  -e SCARF_TOKEN=token `#optional` \
   -e URL=http://localhost:8000 `#optional` \
   -p 8000:8000 \
   -v /path/to/lsio-api/config:/config \

root/app/api.py

@@ -18,9 +18,18 @@
 async def swagger_ui_html():
 	return get_swagger_ui_html(openapi_url="/openapi.json", title="LinuxServer API", swagger_favicon_url="/static/logo.png")
 
+async def get_status():
+    with KeyValueStore() as kv:
+        return kv["status"]
+
 @api.get("/health", summary="Get the health status")
 async def health():
-    return "Success"
+    try:
+        content = await get_status()
+        return JSONResponse(content=content)
+    except Exception:
+        print(traceback.format_exc())
+        raise HTTPException(status_code=404, detail="Not found")
 
 async def get_images():
     with KeyValueStore() as kv:

root/app/models.py

@@ -1,7 +1,8 @@
 from pydantic import BaseModel
 
 # Increment when updating schema or forcing an update on start
-IMAGES_SCHEMA_VERSION = 2
+IMAGES_SCHEMA_VERSION = 3
+SCARF_SCHEMA_VERSION = 1
 
 
 class Tag(BaseModel):
@@ -92,6 +93,7 @@ class Config(BaseModel):
 
 class Image(BaseModel):
     name: str
+    initial_date: str | None = None
     github_url: str
     project_url: str | None = None
     project_logo: str | None = None
@@ -102,6 +104,7 @@ class Image(BaseModel):
     stable: bool
     deprecated: bool
     stars: int
+    monthly_pulls: int | None = None
     tags: list[Tag]
     architectures: list[Architecture]
     changelog: list[Changelog] | None = None

root/app/updater.py

@@ -2,14 +2,18 @@
 from keyvaluestore import KeyValueStore, set_db_schema
 from models import Architecture, Changelog, Tag, EnvVar, Volume, Port, Config
 from models import Custom, SecurityOpt, Device, Cap, Hostname, MacAddress, Image
-from models import Repository, ImagesData, ImagesResponse, IMAGES_SCHEMA_VERSION
+from models import Repository, ImagesData, ImagesResponse, IMAGES_SCHEMA_VERSION, SCARF_SCHEMA_VERSION
 
 import datetime
+import json
 import os
+import requests
 import time
+import traceback
 
 CI = os.environ.get("CI", None)
 INVALIDATE_HOURS = int(os.environ.get("INVALIDATE_HOURS", "24"))
+SCARF_TOKEN = os.environ.get("SCARF_TOKEN", None)
 
 
 def get_tags(readme_vars):
@@ -31,13 +35,18 @@ def get_architectures(readme_vars):
         archs.append(Architecture(arch=item["arch"], tag=item["tag"]))
     return archs
 
-def get_changelogs(readme_vars):
+def get_changelog(readme_vars):
     if "changelogs" not in readme_vars:
-        return None
-    changelogs = []
+        return None, None
+    changelog = []
     for item in readme_vars["changelogs"][0:3]:
-        changelogs.append(Changelog(date=item["date"][0:-1], desc=item["desc"]))
-    return changelogs
+        date = item["date"][0:-1]
+        normalized_date = str(datetime.datetime.strptime(date, "%d.%m.%y").date())
+        changelog.append(Changelog(date=normalized_date, desc=item["desc"]))
+    first_changelog = readme_vars["changelogs"][-1]
+    initial_date = first_changelog["date"][0:-1]
+    normalized_initial_date = str(datetime.datetime.strptime(initial_date, "%d.%m.%y").date())
+    return changelog, normalized_initial_date
 
 def get_description(readme_vars):
     description = readme_vars.get("project_blurb", "No description")
@@ -136,7 +145,7 @@ def get_mac_address(readme_vars):
     hostname = readme_vars.get("param_mac_address", False)
     return MacAddress(mac_address=hostname, desc=readme_vars.get("param_mac_address_desc", ""), optional=optional)
 
-def get_image(repo):
+def get_image(repo, scarf_data):
     print(f"Processing {repo.name}")
     if not repo.name.startswith("docker-") or repo.name.startswith("docker-baseimage-"):
         return None
@@ -153,6 +162,7 @@ def get_image(repo):
     application_setup = None
     if readme_vars.get("app_setup_block_enabled", False):
         application_setup = f"{repo.html_url}?tab=readme-ov-file#application-setup"
+    changelog, initial_date = get_changelog(readme_vars)
     config = Config(
         application_setup=application_setup,
         readonly_supported=readme_vars.get("readonly_supported", None),
@@ -171,8 +181,8 @@ def get_image(repo):
     )
     return Image(
         name=project_name,
+        initial_date=initial_date,
         github_url=repo.html_url,
-        stars=repo.stargazers_count,
         project_url=readme_vars.get("project_url", None),
         project_logo=readme_vars.get("project_logo", None),
         description=get_description(readme_vars),
@@ -181,40 +191,82 @@ def get_image(repo):
         category=categories,
         stable=stable,
         deprecated=deprecated,
+        stars=repo.stargazers_count,
+        monthly_pulls=scarf_data.get(project_name, None),
         tags=tags,
         architectures=get_architectures(readme_vars),
-        changelog=get_changelogs(readme_vars),
+        changelog=changelog,
         config=config,
     )
 
 def update_images():
     with KeyValueStore(invalidate_hours=INVALIDATE_HOURS, readonly=False) as kv:
         is_current_schema = kv.is_current_schema("images", IMAGES_SCHEMA_VERSION)
         if ("images" in kv and is_current_schema) or CI == "1":
-            print(f"{datetime.datetime.now()} - skipped - already updated")
+            print(f"{datetime.datetime.now()} - images skipped - already updated")
             return
         print(f"{datetime.datetime.now()} - updating images")
         images = []
+        scarf_data = json.loads(kv["scarf"])
         repos = gh.get_repos()
         for repo in sorted(repos, key=lambda repo: repo.name):
-            image = get_image(repo)
+            image = get_image(repo, scarf_data)
             if not image:
                 continue
             images.append(image)
 
         data = ImagesData(repositories=Repository(linuxserver=images))
-        last_updated = datetime.datetime.now(datetime.timezone.utc).isoformat(' ', 'seconds')
+        last_updated = datetime.datetime.now(datetime.timezone.utc).isoformat(" ", "seconds")
         response = ImagesResponse(status="OK", last_updated=last_updated, data=data)
         new_state = response.model_dump_json(exclude_none=True)
         kv.set_value("images", new_state, IMAGES_SCHEMA_VERSION)
         print(f"{datetime.datetime.now()} - updated images")
 
+def get_monthly_pulls():
+    pulls_map = {}
+    response = requests.get("https://api.scarf.sh/v2/packages/linuxserver-ci/overview?per_page=1000", headers={"Authorization": f"Bearer {SCARF_TOKEN}"})
+    results = response.json()["results"]
+    for result in results:
+        name = result["package"]["name"].replace("linuxserver/", "")
+        if "total_installs" not in result:
+            continue
+        monthly_pulls = result["total_installs"]
+        pulls_map[name] = monthly_pulls
+    return pulls_map
+
+def update_scarf():
+    with KeyValueStore(invalidate_hours=INVALIDATE_HOURS, readonly=False) as kv:
+        is_current_schema = kv.is_current_schema("scarf", SCARF_SCHEMA_VERSION)
+        if ("scarf" in kv and is_current_schema) or CI == "1":
+            print(f"{datetime.datetime.now()} - scarf skipped - already updated")
+            return
+        print(f"{datetime.datetime.now()} - updating scarf")
+        pulls_map = get_monthly_pulls()
+        if not pulls_map:
+            return
+        new_state = json.dumps(pulls_map)
+        kv.set_value("scarf", new_state, SCARF_SCHEMA_VERSION)
+        print(f"{datetime.datetime.now()} - updated scarf")
+
+def update_status(status):
+    with KeyValueStore(invalidate_hours=0, readonly=False) as kv:
+        print(f"{datetime.datetime.now()} - updating status")
+        kv.set_value("status", status, 0)
+        print(f"{datetime.datetime.now()} - updated status")
+
 def main():
-    set_db_schema()
-    while True:
-        gh.print_rate_limit()
-        update_images()
-        gh.print_rate_limit()
+    try:
+        set_db_schema()
+        while True:
+            gh.print_rate_limit()
+            update_scarf()
+            update_images()
+            gh.print_rate_limit()
+            update_status("Success")
+            time.sleep(INVALIDATE_HOURS*60*60)
+    except:
+        print(traceback.format_exc())
+        update_status("Failed")
         time.sleep(INVALIDATE_HOURS*60*60)
 
 if __name__ == "__main__":