Clean links - remove superfluous tracking elements from URLs

.env.example

@@ -50,3 +50,5 @@ REDLIB_DEFAULT_DISABLE_VISIT_REDDIT_CONFIRMATION=off
 REDLIB_DEFAULT_HIDE_SCORE=off
 # Enable fixed navbar by default
 REDLIB_DEFAULT_FIXED_NAVBAR=on
+# Enable tracking url removal
+REDLIB_DEFAULT_CLEAN_URLS=off

Cargo.toml

@@ -60,6 +60,7 @@ bincode = "1.3.3"
 base2048 = "2.0.2"
 revision = "0.10.0"
 fake_user_agent = "0.2.2"
+clearurls = "0.0.4"
 
 
 [dev-dependencies]

README.md

@@ -34,7 +34,7 @@
    - [Docker](#docker)
      - [Docker Compose](#docker-compose)
      - [Docker CLI](#docker-cli)
-   - Podman 
+   - Podman
       - Quadlets
 
    - [Binary](#binary)
@@ -216,7 +216,7 @@ Stream logs from the Redlib container:
 ```bash
 docker logs -f redlib
 ```
-## Podman 
+## Podman
 
 [Podman](https://podman.io/) lets you run containerized applications in a rootless fashion. Containers are loosely isolated environments that are lightweight and contain everything needed to run the application, so there's no need to rely on what's installed on the host.
 
@@ -225,8 +225,8 @@ Container images for Redlib are available at [quay.io](https://quay.io/repositor
 ### Quadlets
 
 > [!IMPORTANT]
-> These instructions assume that you are on a systemd based distro with [podman](https://podman.io/). If not, follow these [instructions on podman's website](https://podman.io/docs/installation) for how to do so. 
-> It also assumes you have used `loginctl enable-linger <username>` to enable the service to start for your user without logging in. 
+> These instructions assume that you are on a systemd based distro with [podman](https://podman.io/). If not, follow these [instructions on podman's website](https://podman.io/docs/installation) for how to do so.
+> It also assumes you have used `loginctl enable-linger <username>` to enable the service to start for your user without logging in.
 
 Copy the `redlib.container` and `.env.example` files to `.config/containers/systemd/` and modify any relevant values (for example, the ports Redlib should listen on, renaming the .env file and editing its values, etc.).
 
@@ -244,7 +244,7 @@ systemctl --user start redlib.service
 ```
 
 You can check the status of your container by using the following command:
-```bash 
+```bash
 systemctl --user status redlib.service
 ```
 
@@ -441,4 +441,5 @@ Assign a default value for each user-modifiable setting by passing environment v
 | `HIDE_SCORE`                        | `["on", "off"]`                                                                                                                    | `off`         |
 | `HIDE_SIDEBAR_AND_SUMMARY`          | `["on", "off"]`                                                                                                                    | `off`         |
 | `FIXED_NAVBAR`                      | `["on", "off"]`                                                                                                                    | `on`          |
-| `REMOVE_DEFAULT_FEEDS`              | `["on", "off"]`                                                                                                                    | `off`         |
+| `REMOVE_DEFAULT_FEEDS`              | `["on", "off"]`                                                                                                                    | `off`         |
+| `CLEAN_URLS`                        | `["on", "off"]`                                                                                                                    | `off`         |

app.json

@@ -79,6 +79,9 @@
     },
     "REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS": {
       "required": false
+    },
+    "REDLIB_DEFAULT_CLEAN_URLS": {
+      "required": false
     }
   }
 }

src/config.rs

@@ -108,6 +108,9 @@ pub struct Config {
 
 	#[serde(rename = "REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS")]
 	pub(crate) default_remove_default_feeds: Option<String>,
+
+	#[serde(rename = "REDLIB_DEFAULT_CLEAN_URLS")]
+	pub(crate) default_clean_urls: Option<String>,
 }
 
 impl Config {
@@ -156,6 +159,7 @@ impl Config {
 			enable_rss: parse("REDLIB_ENABLE_RSS"),
 			full_url: parse("REDLIB_FULL_URL"),
 			default_remove_default_feeds: parse("REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS"),
+			default_clean_urls: parse("REDLIB_DEFAULT_CLEAN_URLS"),
 		}
 	}
 }
@@ -186,6 +190,7 @@ fn get_setting_from_config(name: &str, config: &Config) -> Option<String> {
 		"REDLIB_ENABLE_RSS" => config.enable_rss.clone(),
 		"REDLIB_FULL_URL" => config.full_url.clone(),
 		"REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS" => config.default_remove_default_feeds.clone(),
+		"REDLIB_DEFAULT_CLEAN_URLS" => config.default_clean_urls.clone(),
 		_ => None,
 	}
 }

src/instance_info.rs

@@ -151,6 +151,7 @@ impl InstanceInfo {
 				["Hide HLS notification", &convert(&self.config.default_hide_hls_notification)],
 				["Subscriptions", &convert(&self.config.default_subscriptions)],
 				["Filters", &convert(&self.config.default_filters)],
+				["Clean URLs", &convert(&self.config.default_clean_urls)],
 			])
 			.with_header_row(["Default preferences"]),
 		);
@@ -186,6 +187,7 @@ impl InstanceInfo {
                     Default blur NSFW: {:?}\n
                     Default use HLS: {:?}\n
                     Default hide HLS notification: {:?}\n
+					Default clean urls: {:?}\n
                     Default subscriptions: {:?}\n
                     Default filters: {:?}\n",
 					self.package_name,
@@ -213,6 +215,7 @@ impl InstanceInfo {
 					self.config.default_blur_nsfw,
 					self.config.default_use_hls,
 					self.config.default_hide_hls_notification,
+					self.config.default_clean_urls,
 					self.config.default_subscriptions,
 					self.config.default_filters,
 				)

src/post.rs

@@ -6,7 +6,8 @@ use crate::config::get_setting;
 use crate::server::RequestExt;
 use crate::subreddit::{can_access_quarantine, quarantine};
 use crate::utils::{
-	error, format_num, get_filters, nsfw_landing, param, parse_post, rewrite_emotes, setting, template, time, val, Author, Awards, Comment, Flair, FlairPart, Post, Preferences,
+	clean_url, error, format_num, get_filters, nsfw_landing, param, parse_post, rewrite_emotes, setting, template, time, val, Author, Awards, Comment, Flair, FlairPart, Post,
+	Preferences,
 };
 use hyper::{Body, Request, Response};
 
@@ -64,7 +65,12 @@ pub async fn item(req: Request<Body>) -> Result<Response<Body>, String> {
 		// Otherwise, grab the JSON output from the request
 		Ok(response) => {
 			// Parse the JSON into Post and Comment structs
-			let post = parse_post(&response[0]["data"]["children"][0]).await;
+			let mut post = parse_post(&response[0]["data"]["children"][0]).await;
+
+			let clean_urls = setting(&req, "clean_urls");
+			if clean_urls == "on".to_owned() {
+				post.media.url = clean_url(post.media.url);
+			}
 
 			let req_url = req.uri().to_string();
 			// Return landing page if this post if this Reddit deems this post

src/search.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::cmp_owned)]
 
 // CRATES
-use crate::utils::{self, catch_random, error, filter_posts, format_num, format_url, get_filters, param, redirect, setting, template, val, Post, Preferences};
+use crate::utils::{self, catch_random, clean_url, error, filter_posts, format_num, format_url, get_filters, param, redirect, setting, template, val, Post, Preferences};
 use crate::{
 	client::json,
 	server::RequestExt,
@@ -131,6 +131,10 @@ pub async fn find(req: Request<Body>) -> Result<Response<Body>, String> {
 	} else {
 		match Post::fetch(&path, quarantined).await {
 			Ok((mut posts, after)) => {
+				let clean_urls = setting(&req, "clean_urls");
+				if clean_urls == "on".to_owned() {
+					posts.iter_mut().for_each(|post| post.media.url = clean_url(post.media.url.clone()));
+				}
 				let (_, all_posts_filtered) = filter_posts(&mut posts, &filters);
 				let no_posts = posts.is_empty();
 				let all_posts_hidden_nsfw = !no_posts && (posts.iter().all(|p| p.flags.nsfw) && setting(&req, "show_nsfw") != "on");

src/settings.rs

@@ -24,7 +24,7 @@ struct SettingsTemplate {
 
 // CONSTANTS
 
-const PREFS: [&str; 19] = [
+const PREFS: [&str; 20] = [
 	"theme",
 	"front_page",
 	"layout",
@@ -44,6 +44,7 @@ const PREFS: [&str; 19] = [
 	"disable_visit_reddit_confirmation",
 	"video_quality",
 	"remove_default_feeds",
+	"clean_urls",
 ];
 
 // FUNCTIONS

src/subreddit.rs

@@ -3,8 +3,8 @@
 use crate::{config, utils};
 // CRATES
 use crate::utils::{
-	catch_random, error, filter_posts, format_num, format_url, get_filters, info, nsfw_landing, param, redirect, rewrite_urls, setting, template, val, Post, Preferences,
-	Subreddit,
+	catch_random, clean_url, error, filter_posts, format_num, format_url, get_filters, info, nsfw_landing, param, redirect, rewrite_urls, setting, template, val, Post,
+	Preferences, Subreddit,
 };
 use crate::{client::json, server::RequestExt, server::ResponseExt};
 use askama::Template;
@@ -164,6 +164,10 @@ pub async fn community(req: Request<Body>) -> Result<Response<Body>, String> {
 	} else {
 		match Post::fetch(&path, quarantined).await {
 			Ok((mut posts, after)) => {
+				let clean_urls = setting(&req, "clean_urls");
+				if clean_urls == "on".to_owned() {
+					posts.iter_mut().for_each(|post| post.media.url = clean_url(post.media.url.clone()));
+				}
 				let (_, all_posts_filtered) = filter_posts(&mut posts, &filters);
 				let no_posts = posts.is_empty();
 				let all_posts_hidden_nsfw = !no_posts && (posts.iter().all(|p| p.flags.nsfw) && setting(&req, "show_nsfw") != "on");

src/utils.rs

@@ -7,12 +7,13 @@ use crate::config::{self, get_setting};
 //
 use crate::{client::json, server::RequestExt};
 use askama::Template;
+use clearurls::UrlCleaner;
 use cookie::Cookie;
 use hyper::{Body, Request, Response};
 use libflate::deflate::{Decoder, Encoder};
 use log::error;
 use regex::Regex;
-use revision::revisioned;
+use revision::{revisioned, Error};
 use rust_embed::RustEmbed;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use serde_json::Value;
@@ -22,7 +23,7 @@ use std::env;
 use std::io::{Read, Write};
 use std::str::FromStr;
 use std::string::ToString;
-use std::sync::LazyLock;
+use std::sync::{Arc, LazyLock};
 use time::{macros::format_description, Duration, OffsetDateTime};
 use url::Url;
 
@@ -621,7 +622,7 @@ pub struct Params {
 }
 
 #[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq)]
-#[revisioned(revision = 1)]
+#[revisioned(revision = 2)]
 pub struct Preferences {
 	#[revision(start = 1)]
 	#[serde(skip_serializing, skip_deserializing)]
@@ -670,6 +671,8 @@ pub struct Preferences {
 	pub hide_score: String,
 	#[revision(start = 1)]
 	pub remove_default_feeds: String,
+	#[revision(start = 2, default_fn = "default_clean_urls")]
+	pub clean_urls: String,
 }
 
 fn serialize_vec_with_plus<S>(vec: &[String], serializer: S) -> Result<S::Ok, S::Error>
@@ -728,6 +731,7 @@ impl Preferences {
 			hide_awards: setting(req, "hide_awards"),
 			hide_score: setting(req, "hide_score"),
 			remove_default_feeds: setting(req, "remove_default_feeds"),
+			clean_urls: setting(req, "clean_urls"),
 		}
 	}
 
@@ -744,6 +748,9 @@ impl Preferences {
 	pub fn to_bincode_str(&self) -> Result<String, String> {
 		Ok(base2048::encode(&self.to_compressed_bincode()?))
 	}
+	fn default_clean_urls(_revision: u16) -> Result<String, Error> {
+		Ok("off".to_owned())
+	}
 }
 
 pub fn deflate_compress(i: Vec<u8>) -> Result<Vec<u8>, String> {
@@ -1075,6 +1082,24 @@ pub fn format_url(url: &str) -> String {
 	}
 }
 
+// Remove tracking query params
+static URL_CLEANER: LazyLock<Arc<UrlCleaner>> = LazyLock::new(|| Arc::new(UrlCleaner::from_embedded_rules().expect("Failed to initialize UrlCleaner")));
+
+pub fn clean_url(url: String) -> String {
+	let is_external_url = match Url::parse(url.as_str()) {
+		Ok(parsed_url) => parsed_url.domain().is_some(),
+		_ => false,
+	};
+	let mut cleaned_url = url.clone();
+	if is_external_url {
+		cleaned_url = match URL_CLEANER.clear_single_url_str(cleaned_url.as_str()) {
+			Ok(cleaned_result) => cleaned_result.as_ref().to_owned(),
+			_ => cleaned_url,
+		}
+	}
+	cleaned_url
+}
+
 static REGEX_BULLET: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?m)^- (.*)$").unwrap());
 static REGEX_BULLET_CONSECUTIVE_LINES: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"</ul>\n<ul>").unwrap());
 
@@ -1541,10 +1566,11 @@ mod tests {
 			hide_awards: "off".to_owned(),
 			hide_score: "off".to_owned(),
 			remove_default_feeds: "off".to_owned(),
+			clean_urls: "off".to_owned(),
 		};
 		let urlencoded = serde_urlencoded::to_string(prefs).expect("Failed to serialize Prefs");
 
-		assert_eq!(urlencoded, "theme=laserwave&front_page=default&layout=compact&wide=on&blur_spoiler=on&show_nsfw=off&blur_nsfw=on&hide_hls_notification=off&video_quality=best&hide_sidebar_and_summary=off&use_hls=on&autoplay_videos=on&fixed_navbar=on&disable_visit_reddit_confirmation=on&comment_sort=confidence&post_sort=top&subscriptions=memes%2Bmildlyinteresting&filters=&hide_awards=off&hide_score=off&remove_default_feeds=off");
+		assert_eq!(urlencoded, "theme=laserwave&front_page=default&layout=compact&wide=on&blur_spoiler=on&show_nsfw=off&blur_nsfw=on&hide_hls_notification=off&video_quality=best&hide_sidebar_and_summary=off&use_hls=on&autoplay_videos=on&fixed_navbar=on&disable_visit_reddit_confirmation=on&comment_sort=confidence&post_sort=top&subscriptions=memes%2Bmildlyinteresting&filters=&hide_awards=off&hide_score=off&remove_default_feeds=off&clean_urls=off");
 	}
 }
 
@@ -1657,6 +1683,7 @@ static KNOWN_GOOD_CONFIGS: &[&str] = &[
 	"ఴӅβØØҞÉဏႢձĬ༧ȒʯऌԔӵ୮༏",
 	"ਧՊΥÀÃǎƱГ۸ඣമĖฤ႙ʟาúໜϾௐɥঀĜໃહཞઠѫҲɂఙ࿔DzઉƲӟӻĻฅΜδ໖ԜǗဖငƦơ৶Ą௩ԹʛใЛʃශаΏ",
 	"ਧԩΥÀÃΊ౭൩ඔႠϼҭöҪƸռઇԾॐნɔາǒՍҰच௨ಖມŃЉŐདƦ๙ϩএఠȝഽйʮჯඒϰळՋ௮ສ৵ऎΦѧਹಧଟƙŃ३î༦ŌပղयƟแҜ།",
+	"ԅҫടÀÁÓɿဧඑปฎɣĊဨ۹ÕќଌઞೱũდϚӕӜĄӑƺӌĩҹஸইພÇƄŴ࿀টŨ୬ਦတႣಮѿౡಛളƖljljೱ႒ઽරਊԚƢൿɧűХխபծಥဤ౧À",
 ];
 
 #[test]

templates/settings.html

@@ -145,6 +145,12 @@
 					<input type="checkbox" name="disable_visit_reddit_confirmation" {% if
 						prefs.disable_visit_reddit_confirmation=="on" %}checked{% endif %}>
 				</div>
+				<div class="prefs-group">
+					<label for="clean_urls">Remove tracking elements from URLs</label>
+					<input type="hidden" value="off" name="clean_urls">
+					<input type="checkbox" name="clean_urls" {% if
+						prefs.clean_urls=="on" %}checked{% endif %}>
+				</div>
 			</fieldset>
 			<input id="save" type="submit" value="Save">
 		</div>
@@ -214,4 +220,4 @@
 	</div>
 </div>
 
-{% endblock %}
+{% endblock %}