Add a flake with a NixOS module to generate files

flake.nix

@@ -0,0 +1,91 @@
+{
+  description = "nixpkgs-swh";
+
+  outputs = { self, nixpkgs }:
+  let
+    system = "x86_64-linux";
+    pkgs = import nixpkgs {
+      system = "x86_64-linux";
+      overlays = [ self.overlay ];
+    };
+  in {
+    overlay = final: prev: {
+      nixpkgs-swh-generate = let
+        binPath = with final; lib.makeBinPath [ curl nix python3 jq pandoc ];
+      in final.stdenv.mkDerivation {
+        name = "nixpkgs-swh-generate";
+        dontUnpack = true;
+        nativeBuildInputs = [ final.makeWrapper ];
+        installPhase = ''
+          mkdir -p $out/bin
+          cp ${./scripts/generate.sh} $out/bin/nixpkgs-swh-generate
+          substituteInPlace $out/bin/nixpkgs-swh-generate \
+            --replace-fail './scripts/swh-urls.nix' '${./scripts/swh-urls.nix}' \
+            --replace-fail './scripts/add-sri.py' '${./scripts/add-sri.py}' \
+            --replace-fail './scripts/analyze.py' '${./scripts/analyze.py}' \
+            --replace-fail '$PWD/scripts/find-tarballs.nix' '${./scripts/find-tarballs.nix}'
+          wrapProgram $out/bin/nixpkgs-swh-generate \
+            --prefix PATH : ${binPath}
+        '';
+      };
+    };
+
+    packages.x86_64-linux.nixpkgs-swh-generate = pkgs.nixpkgs-swh-generate;
+    defaultPackage.x86_64-linux = pkgs.nixpkgs-swh-generate;
+
+    nixosModules.nixpkgs-swh = { config, pkgs, lib, ... }: let
+      cfg = config.services.nixpkgs-swh;
+      dir = "/var/lib/nixpkgs-swh";
+    in {
+      options = {
+        services.nixpkgs-swh = {
+          enable = lib.mkOption {
+            type = lib.types.bool;
+            default = false;
+            description = ''
+              Whether to run the nixpkgs-swh service.
+            '';
+          };
+          testing = lib.mkOption {
+            type = lib.types.bool;
+            default = false;
+            description = ''
+              Whether to only evaluate the hello attribute for testing purpose.
+            '';
+          };
+          fqdn = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The Nginx vhost FQDN used to serve built files.
+            '';
+          };
+        };
+      };
+      config = lib.mkIf cfg.enable {
+        nixpkgs.overlays = [ self.overlay ];
+        systemd.services.nixpkgs-swh = {
+          description = "nixpkgs-swh";
+          wantedBy = [ "multi-user.target" ];
+          # Do it every day
+          startAt = "daily";
+          script = ''
+            ${pkgs.nixpkgs-swh-generate}/bin/nixpkgs-swh-generate ${dir} ${if cfg.testing then "true" else "false"} unstable
+          '';
+        };
+        systemd.timers.nixpkgs-swh.timerConfig = {
+          Persistent = true;
+        };
+        services.nginx.virtualHosts = {
+          "${cfg.fqdn}" = {
+            locations."/" = {
+              root = "${dir}";
+              extraConfig = ''
+                autoindex on;
+              '';
+            };
+          };
+        };
+      };
+    };
+  };
+}

scripts/analyze.py

@@ -5,6 +5,7 @@
 import json
 import sys
 import re
+import datetime
 from urllib.parse import urlparse
 
 with open(sys.argv[1], "r") as read_file:
@@ -62,17 +63,19 @@
             e['file-type'] = v
             break
 
+today = datetime.date.today()
 
 readme = """
 
-The file [`sources-{release}-full.json`](https://nix-community.github.io/nixpkgs-swh/sources-{release}-full.json)
+The file [`sources-{release}-full.json`](./sources-{release}-full.json)
 has been built from the [nixpkgs revision
 `{revision}`](https://github.com/NixOS/nixpkgs/tree/{revision}).
 This file contains `{sourceNumber}` sources, coming from
 `{hostNumber}` different hosts.
 
-The file [`sources-{release}.json`](https://nix-community.github.io/nixpkgs-swh/sources-{release}.json) is a filtered version which only contains archives. This file is consumed by SWH.
+The file [`sources-{release}.json`](./sources-{release}.json) is a filtered version which only contains archives. This file is consumed by Software Heritage to fill its archive.
 
+Generated the {today}.
 """
 
 sortedHosts = sorted(hosts.items(), key=lambda h: h[1], reverse=True)
@@ -81,7 +84,8 @@
     revision=j['revision'],
     release=j['release'],
     sourceNumber=len(sources),
-    hostNumber=len(sortedHosts)))
+    hostNumber=len(sortedHosts),
+    today=today))
 
 
 print("\n#### By host\n")

scripts/generate.sh

@@ -1,7 +1,4 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash
-#!nix-shell -I nixpkgs=./nix
-#!nix-shell -p nix git openssh python3 curl jq
+#!/usr/bin/env bash
 set -euo pipefail
 
 
@@ -49,6 +46,8 @@ generate-release() {
         --argstr release ${RELEASE} \
         --argstr evaluation ${EVAL_ID} \
         --argstr timestamp $(date +%s) \
+        --argstr find-tarballs $PWD/scripts/find-tarballs.nix \
+        --arg testing $TESTING \
         > ${SOURCES_FILE_FULL}
 
     echo "*** Add integrity attribute"
@@ -66,23 +65,28 @@ generate-release() {
 
 generate-readme() {
     cat <<EOF > ${DEST_DIR}/README.md
-Fill the Software Heritage archive
+# Fill the Software Heritage archive
 
 EOF
 
-for i in $@; do
-    generate-release ${i}
-    echo "### NixOS \`${i}\`" >> ${DEST_DIR}/README.md
-    cat ${DEST_DIR}/readme-${i}.md >> ${DEST_DIR}/README.md
-    echo >> ${DEST_DIR}/README.md
-    echo >> ${DEST_DIR}/README.md
-    shift
-done
+    for i in $@; do
+        generate-release ${i}
+        echo "### NixOS \`${i}\` release" >> ${DEST_DIR}/README.md
+        cat ${DEST_DIR}/readme-${i}.md >> ${DEST_DIR}/README.md
+        echo >> ${DEST_DIR}/README.md
+        echo >> ${DEST_DIR}/README.md
+        shift
+    done
 }
 
-
 DEST_DIR=$1
 mkdir -p ${DEST_DIR}
 shift
 
+TESTING=$1
+shift
+
 generate-readme $@
+
+echo "** Converting README.md to index.html"
+pandoc ${DEST_DIR}/README.md > ${DEST_DIR}/index.html

scripts/swh-urls.nix

@@ -1,12 +1,12 @@
-{ revision ? null, release ? null, evaluation ? null, timestamp ? null }:
+{ revision ? null, release ? null, evaluation ? null, timestamp ? null, testing ? false, find-tarballs }:
 
 with builtins;
 let
   pkgs = import <nixpkgs> {};
   mirrors = import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>;
   expr =  import <nixpkgs/maintainers/scripts/all-tarballs.nix>;
-  urls = import ./find-tarballs.nix {expr = expr;};
-  
+  urls = import find-tarballs {expr = if testing then expr.hello else expr;};
+
   # This is avoid double slashes in urls that make url non valid
   concatUrls = a: b: (pkgs.lib.removeSuffix "/" a) + "/" + (pkgs.lib.removePrefix "/" b);
 
@@ -20,7 +20,7 @@ let
   in if isMirrorUrl
   then [ url ]
   else map (r: concatUrls r path) resolvedUrls;
-  
+
   # Transform the url list to swh format
   toSwh = s: {
     inherit (s) postFetch outputHashMode outputHashAlgo outputHash;