parser: Update nginx to better match F=1 behavior

taoky · taoky · commit d4e58702f7b5 · 2024-08-23T00:05:51.000+08:00
With testcase from mirror.ghettoforge.org/distributions/gf/
diff --git a/fixtures/ghettoforge/index.html b/fixtures/ghettoforge/index.html
@@ -0,0 +1,18 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<html>
+ <head>
+  <title>Index of /distributions/gf</title>
+ </head>
+ <body>
+<h1>Index of /distributions/gf</h1>
+<pre><img src="/icons/blank.gif" alt="Icon "> <a href="?C=N;O=D">Name</a>                                <a href="?C=M;O=A">Last modified</a>      <a href="?C=S;O=A">Size</a>  <a href="?C=D;O=A">Description</a><hr><img src="/icons/back.gif" alt="[PARENTDIR]"> <a href="/distributions/">Parent Directory</a>                                         -   
+<img src="/icons/unknown.gif" alt="[   ]"> <a href="RPM-GPG-KEY-gf.el7">RPM-GPG-KEY-gf.el7</a>                  2014-12-30 02:53  3.0K  
+<img src="/icons/unknown.gif" alt="[   ]"> <a href="RPM-GPG-KEY-gf.el8">RPM-GPG-KEY-gf.el8</a>                  2020-01-13 09:40  3.1K  
+<img src="/icons/unknown.gif" alt="[   ]"> <a href="RPM-GPG-KEY-gf.el9">RPM-GPG-KEY-gf.el9</a>                  2022-08-03 11:28  1.6K  
+<img src="/icons/folder.gif" alt="[DIR]"> <a href="archive/">archive/</a>                            2020-12-21 02:34    -   
+<img src="/icons/folder.gif" alt="[DIR]"> <a href="el/">el/</a>                                 2022-08-02 11:57    -   
+<img src="/icons/unknown.gif" alt="[   ]"> <a href="gf-release-latest.gf.el7.noarch.rpm">gf-release-latest.gf.el7.noarch.rpm</a> 2021-08-21 10:38  8.0K  
+<img src="/icons/unknown.gif" alt="[   ]"> <a href="gf-release-latest.gf.el8.noarch.rpm">gf-release-latest.gf.el8.noarch.rpm</a> 2021-08-21 10:39   11K  
+<img src="/icons/unknown.gif" alt="[   ]"> <a href="gf-release-latest.gf.el9.noarch.rpm">gf-release-latest.gf.el9.noarch.rpm</a> 2022-08-03 12:16  9.2K  
+<hr></pre>
+</body></html>
diff --git a/src/parser/README.md b/src/parser/README.md
@@ -6,7 +6,7 @@ This is a list of parsers that tsumugu supports:
 - directory_lister: [Directory Lister](https://www.directorylister.com/).
 - docker: A specialized parser for <https://download.docker.com/>.
 - lighttpd: [lighttpd's mod_dirlisting](https://redmine.lighttpd.net/projects/lighttpd/wiki/Docs_ModDirlisting).
-- nginx: [Nginx's autoindex](https://nginx.org/en/docs/http/ngx_http_autoindex_module.html).
+- nginx: [Nginx's autoindex](https://nginx.org/en/docs/http/ngx_http_autoindex_module.html). It should also work with Apache2's autoindex `F=1` mode.
 - caddy: [Caddy's file_server](https://caddyserver.com/docs/caddyfile/directives/file_server).
 - fancyindex: [Nginx fancyindex](https://github.com/aperezdc/ngx-fancyindex).
 
diff --git a/src/parser/fancyindex.rs b/src/parser/fancyindex.rs
@@ -13,14 +13,6 @@ use scraper::{Html, Selector};
 #[derive(Debug, Clone, Default)]
 pub struct FancyIndexListingParser;
 
-fn guess_date_fmt(date: &str) -> String {
-    let two_colons = contains_two_colons(date);
-    let abbr_month = contains_abbreviated_month(date);
-    let dfmt = if abbr_month { "%Y-%b-%d" } else { "%Y-%m-%d" };
-    let tfmt = if two_colons { "%H:%M:%S" } else { "%H:%M" };
-    format!("{} {}", dfmt, tfmt)
-}
-
 impl Parser for FancyIndexListingParser {
     fn get_list(&self, client: &Client, url: &Url) -> Result<ListResult> {
         let resp = get(client, url.clone())?;
@@ -56,7 +48,7 @@ impl Parser for FancyIndexListingParser {
             let date = date.trim();
 
             // decide (guess) which time format to use
-            let date_fmt = guess_date_fmt(date);
+            let (date_fmt, _) = guess_date_fmt(date);
             let date = NaiveDateTime::parse_from_str(date, &date_fmt)?;
 
             items.push(ListItem::new(
@@ -151,9 +143,4 @@ mod tests {
             _ => unreachable!(),
         }
     }
-
-    #[test]
-    fn test_guess_date_fmt() {
-        assert_eq!(guess_date_fmt("2024-Jul-15 09:46"), "%Y-%b-%d %H:%M");
-    }
 }
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
@@ -80,3 +80,45 @@ fn contains_abbreviated_month(s: &str) -> bool {
 fn contains_two_colons(s: &str) -> bool {
     s.matches(':').count() >= 2
 }
+
+fn has_numeric_prefix(s: &str) -> bool {
+    s.chars().take(4).all(|c| c.is_ascii_digit()) && s.len() >= 4
+}
+
+// Returns format and regex string
+fn guess_date_fmt(date: &str) -> (String, String) {
+    let two_colons = contains_two_colons(date);
+    let abbr_month = contains_abbreviated_month(date);
+    let year_first = has_numeric_prefix(date);
+    let (dfmt, dfmt_regex) = match (abbr_month, year_first) {
+        (true, true) => ("%Y-%b-%d", r"\d{4}-\w{3}-\d{2}"),
+        (true, false) => ("%d-%b-%Y", r"\d{2}-\w{3}-\d{4}"),
+        (false, true) => ("%Y-%m-%d", r"\d{4}-\d{2}-\d{2}"),
+        (false, false) => ("%d-%m-%Y", r"\d{2}-\d{2}-\d{4}"),
+    };
+    let (tfmt, tfmt_regex) = if two_colons {
+        ("%H:%M:%S", r"\d{2}:\d{2}:\d{2}")
+    } else {
+        ("%H:%M", r"\d{2}:\d{2}")
+    };
+    (
+        format!("{} {}", dfmt, tfmt),
+        format!("{} {}", dfmt_regex, tfmt_regex),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_guess_date_fmt() {
+        assert_eq!(
+            guess_date_fmt("2024-Jul-15 09:46"),
+            (
+                "%Y-%b-%d %H:%M".to_owned(),
+                r"\d{4}-\w{3}-\d{2} \d{2}:\d{2}".to_owned()
+            )
+        );
+    }
+}
diff --git a/src/parser/nginx.rs b/src/parser/nginx.rs
@@ -11,19 +11,8 @@ use super::*;
 use anyhow::{anyhow, Result};
 use regex::Regex;
 
-#[derive(Debug, Clone)]
-pub struct NginxListingParser {
-    metadata_regex: Regex,
-}
-
-impl Default for NginxListingParser {
-    fn default() -> Self {
-        Self {
-            metadata_regex: Regex::new(r"(\d{2}-\w{3}-\d{4} \d{2}:\d{2})\s+([\d\.\-kMG]+)$")
-                .unwrap(),
-        }
-    }
-}
+#[derive(Debug, Clone, Default)]
+pub struct NginxListingParser {}
 
 impl Parser for NginxListingParser {
     fn get_list(&self, client: &reqwest::blocking::Client, url: &url::Url) -> Result<ListResult> {
@@ -34,11 +23,17 @@ impl Parser for NginxListingParser {
         let document = Html::parse_document(&body);
         let selector = Selector::parse("a").unwrap();
         let mut items = Vec::new();
+        let mut date_fmt = None;
+        let mut date_regex = None;
         for element in document.select(&selector) {
             let href = match element.value().attr("href") {
                 Some(href) => href,
                 None => continue,
             };
+            if href.starts_with('?') {
+                // Apache autoindex commands, skip.
+                continue;
+            }
             // It's not proper to get filename by <a> text
             // As when it is too long, this could happen:
             // ceph-immutable-object-cache_17.2.6-pve1+3_amd64..> 03-May-2023 23:52              150048
@@ -55,6 +50,11 @@ impl Parser for NginxListingParser {
             if name == ".." {
                 continue;
             }
+            // extra check for Apache server
+            let inner = element.inner_html();
+            if inner == "Parent Directory" {
+                continue;
+            }
             let type_ = if href.as_str().ends_with('/') {
                 FileType::Directory
             } else {
@@ -69,12 +69,25 @@ impl Parser for NginxListingParser {
                 .to_string();
             let metadata_raw = metadata_raw.trim();
             debug!("{:?}", metadata_raw);
-            let metadata = self.metadata_regex.captures(metadata_raw).ok_or(anyhow!(
-                "Get '{}' for metadata, is this a nginx page?",
-                metadata_raw
-            ))?;
+            // guess date format...
+            if date_fmt.is_none() {
+                let (f, r) = guess_date_fmt(metadata_raw);
+                date_fmt = Some(f);
+                date_regex = Some(Regex::new(&format!(r"({})\s+([\d\.\-kKMG]+)$", r))?);
+                debug!("date_fmt: {:?} date_regex: {:?}", date_fmt, date_regex)
+            }
+            let metadata = date_regex
+                .clone()
+                .unwrap()
+                .captures(metadata_raw)
+                .ok_or(anyhow!(
+                    "Get '{}' for {} ({}) metadata, is this a nginx page?",
+                    metadata_raw,
+                    name,
+                    href
+                ))?;
             let date = metadata.get(1).unwrap().as_str();
-            let date = NaiveDateTime::parse_from_str(date, "%d-%b-%Y %H:%M")?;
+            let date = NaiveDateTime::parse_from_str(date, &date_fmt.clone().unwrap())?;
             let size = metadata.get(2).unwrap().as_str();
             debug!("{} {} {:?} {} {:?}", href, name, type_, date, size);
             items.push(ListItem::new(
@@ -84,7 +97,11 @@ impl Parser for NginxListingParser {
                 {
                     if size == "-" {
                         None
-                    } else if size.contains('k') || size.contains('M') || size.contains('G') {
+                    } else if size.contains('k')
+                        || size.contains('K')
+                        || size.contains('M')
+                        || size.contains('G')
+                    {
                         let (n_size, unit) = FileSize::get_humanized(size);
                         Some(FileSize::HumanizedBinary(n_size, unit))
                     } else {
@@ -101,8 +118,11 @@ impl Parser for NginxListingParser {
 
 #[cfg(test)]
 mod tests {
+    use test_log::test;
     use url::Url;
 
+    use crate::listing::SizeUnit;
+
     use super::*;
 
     #[test]
@@ -182,4 +202,38 @@ mod tests {
             _ => unreachable!(),
         }
     }
+
+    #[test]
+    fn test_ghettoforge() {
+        let client = reqwest::blocking::Client::new();
+        let items = NginxListingParser::default()
+            .get_list(
+                &client,
+                &url::Url::parse("http://localhost:1921/ghettoforge").unwrap(),
+            )
+            .unwrap();
+        match items {
+            ListResult::List(items) => {
+                assert_eq!(items.len(), 8);
+                assert_eq!(items[0].name, "RPM-GPG-KEY-gf.el7");
+                assert_eq!(items[0].type_, FileType::File);
+                assert_eq!(
+                    items[0].size,
+                    Some(FileSize::HumanizedBinary(3.0, SizeUnit::K))
+                );
+                assert_eq!(
+                    items[0].mtime,
+                    NaiveDateTime::parse_from_str("2014-12-30 02:53", "%Y-%m-%d %H:%M").unwrap()
+                );
+                assert_eq!(items[3].name, "archive");
+                assert_eq!(items[3].type_, FileType::Directory);
+                assert_eq!(items[3].size, None);
+                assert_eq!(
+                    items[3].mtime,
+                    NaiveDateTime::parse_from_str("2020-12-21 02:34", "%Y-%m-%d %H:%M").unwrap()
+                );
+            }
+            _ => unreachable!(),
+        }
+    }
 }
