From dad1cbc87af47d71c887718e013b1e1614b10b26 Mon Sep 17 00:00:00 2001
From: Jason Ish <jason@codemonkey.net>
Date: Fri, 12 Jul 2024 14:14:16 -0600
Subject: [PATCH] sqlite: dns array handling

- treat dns.rrname and dns.queries.rrname a the same for query and
  aggregations
- flatten dns.queries and dns.arrays, allowing queries like
  "dns.queries.rrname:evebox.org" much like Elastic does.
- for a key of "dns.additionals" and "dns.authorities", just do a glob
  query for now. Elastic can't do anything like this

Issue: https://github.com/jasonish/evebox/issues/317
---
 src/sqlite/builder.rs          | 71 ++++++++++++++++++++++++++++++++--
 src/sqlite/eventrepo/events.rs |  3 +-
 2 files changed, 70 insertions(+), 4 deletions(-)

diff --git a/src/sqlite/builder.rs b/src/sqlite/builder.rs
index 0fb3dc3d..f1fe83e7 100644
--- a/src/sqlite/builder.rs
+++ b/src/sqlite/builder.rs
@@ -15,6 +15,7 @@ pub(crate) struct EventQueryBuilder<'a> {
 
     select: Vec<String>,
     from: Vec<String>,
+    left_join: Vec<String>,
     wheres: Vec<String>,
     group_by: Vec<String>,
     order_by: Option<(String, String)>,
@@ -147,6 +148,40 @@ impl<'a> EventQueryBuilder<'a> {
         Ok(self)
     }
 
+    pub fn add_left_join(&mut self, sql: String) {
+        if !self.left_join.contains(&sql) {
+            self.left_join.push(sql);
+        }
+    }
+
+    pub fn left_join_from_query_string(
+        &mut self,
+        q: &'a [queryparser::QueryElement],
+    ) -> Result<(), sqlx::error::BoxDynError> {
+        for e in q {
+            match &e.value {
+                queryparser::QueryValue::KeyValue(k, _v) => {
+                    //if k == "dns.rrname" || k == "dns.queries.rrname" {
+                    if k == "dns.rrname" || k.starts_with("dns.queries") {
+                        self.add_left_join(
+                            "LEFT JOIN json_each(events.source, '$.dns.queries') AS _dns_queries"
+                                .to_string(),
+                        );
+                    } else if k.starts_with("dns.answers.") {
+                        self.add_left_join(
+                            "LEFT JOIN json_each(events.source, '$.dns.answers') AS _dns_answers"
+                                .to_string(),
+                        );
+                    }
+                }
+                queryparser::QueryValue::String(_) => {}
+                queryparser::QueryValue::From(_) => {}
+                queryparser::QueryValue::To(_) => {}
+            }
+        }
+        Ok(())
+    }
+
     pub fn apply_query_string(
         &mut self,
         q: &'a [queryparser::QueryElement],
@@ -207,10 +242,36 @@ impl<'a> EventQueryBuilder<'a> {
                                     self.push_where("(events.source->>'dns'->>'type' = 'query' OR events.source->>'dns'->>'type' = 'request')");
                                 } else if k == "dns.type" && (v == "response" || v == "answer") {
                                     self.push_where("(events.source->>'dns'->>'type' = 'answer' OR events.source->>'dns'->>'type' = 'response')");
-                                } else if k == "dns.rrname" {
-                                    self.from("json_each(events.source->>'dns'->>'queries') AS dns_queries");
-                                    self.push_where("dns_queries.value->>'rrname' = ?");
+                                } else if k == "dns.rrname" || k == "dns.queries.rrname" {
+                                    self.push_where("(events.source->>'dns'->>'rrname' = ? OR _dns_queries.value->>'rrname' = ?)");
                                     self.push_arg(v)?;
+                                    self.push_arg(v)?;
+                                } else if k.starts_with("dns.queries.") {
+                                    let path: String = k
+                                        .split('.')
+                                        .skip(2)
+                                        .map(|p| format!("'{}'", p))
+                                        .collect::<Vec<String>>()
+                                        .join("->>");
+                                    self.push_where(format!("_dns_queries.value->>{} = ?", path));
+                                    self.push_arg(v)?;
+                                } else if k.starts_with("dns.answers.") {
+                                    let path: String = k
+                                        .split('.')
+                                        .skip(2)
+                                        .map(|p| format!("'{}'", p))
+                                        .collect::<Vec<String>>()
+                                        .join("->>");
+                                    self.push_where(format!("_dns_answers.value->>{} = ?", path));
+                                    self.push_arg(v)?;
+                                } else if k.starts_with("dns.authorities") {
+                                    // Lazy helper - can't be done with Elastic though.
+                                    self.push_where("events.source->>'dns'->>'authorities' GLOB ?");
+                                    self.push_arg(format!("*{}*", v))?;
+                                } else if k.starts_with("dns.additionals") {
+                                    // Lazy helper - can't be done with Elastic though.
+                                    self.push_where("events.source->>'dns'->>'additionals' GLOB ?");
+                                    self.push_arg(format!("*{}*", v))?;
                                 } else {
                                     self.where_source_json_extract(k, "=", v)?;
                                 }
@@ -272,6 +333,10 @@ impl<'a> EventQueryBuilder<'a> {
         sql.push_str(" from ");
         sql.push_str(&self.from.join(", "));
 
+        for left_join in &self.left_join {
+            sql.push_str(&format!(" {}", left_join));
+        }
+
         if !self.fts_phrases.is_empty() {
             let query = self.fts_phrases.join(" AND ");
             self.push_where("events.rowid in (select rowid from fts where fts match ?)");
diff --git a/src/sqlite/eventrepo/events.rs b/src/sqlite/eventrepo/events.rs
index 1948768b..4bfeaded 100644
--- a/src/sqlite/eventrepo/events.rs
+++ b/src/sqlite/eventrepo/events.rs
@@ -20,11 +20,12 @@ impl SqliteEventRepo {
     ) -> Result<serde_json::Value, DatastoreError> {
         let mut builder = EventQueryBuilder::new(self.fts().await);
         builder
-            .select("events.rowid AS id")
+            .select("DISTINCT(events.rowid) AS id")
             .select("events.archived AS archived")
             .select("events.escalated AS escalated")
             .select("events.source AS source");
         builder.from("events");
+        builder.left_join_from_query_string(&options.query_string)?;
         builder.limit(500);
 
         if let Some(event_type) = options.event_type {