Optimization

Author: Your Name

.devcontainer/Dockerfile

@@ -2,6 +2,7 @@ FROM ubuntu:22.04 as rbase
 RUN apt-get update && \
     apt-get install -y \
     git \
+    git-lfs \
     curl \
     build-essential
 

.gitattributes

@@ -0,0 +1,2 @@
+data/Image-arm64be filter=lfs diff=lfs merge=lfs -text
+data/Image-arm32le filter=lfs diff=lfs merge=lfs -text

.vscode/tasks.json

@@ -11,9 +11,37 @@
             }
         },
         {
-            "label": "Run",
+            "label": "Build - Release",
             "type": "shell",
-            "command": "cargo run -- ${workspaceFolder}/data/Image-arm32",
+            "command": "cargo build --release",
+            "group": "build",
+            "options": {
+                "cwd": "${workspaceFolder}"
+            }
+        },
+        {
+            "label": "Run 32 - Debug",
+            "type": "shell",
+            "command": "cargo run -- ${workspaceFolder}/data/Image-arm32le",
+            "group": "build",
+            "options": {
+                "cwd": "${workspaceFolder}",
+                "env": { "RUST_BACKTRACE": "1" }
+            }
+        },
+        {
+            "label": "Run 32 - Release",
+            "type": "shell",
+            "command": "cargo run --release -- ${workspaceFolder}/data/Image-arm32le",
+            "group": "build",
+            "options": {
+                "cwd": "${workspaceFolder}"
+            }
+        },
+        {
+            "label": "Run 64 - Release",
+            "type": "shell",
+            "command": "cargo run --release -- --64 --big ${workspaceFolder}/data/Image-arm64be",
             "group": "build",
             "options": {
                 "cwd": "${workspaceFolder}"
@@ -29,4 +57,4 @@
             }
         },
     ]
-}
+}

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "rbase"
-version = "0.1.2"
+version = "0.1.3"
 edition = "2021"
 repository = "https://github.com/WorksButNotTested/rbase.git"
 authors = [ "WorksButNotTested" ]
@@ -37,6 +37,7 @@ install-updater = false
 
 [dependencies]
 clap = { version = "4.5.4", features = ["derive"] }
+dashmap = { version = "5.5.3", features = ["rayon"] }
 indicatif = { version = "0.17.8", features = ["rayon"] }
 memmap2 = "0.9.4"
 num_cpus = "1.16.0"

Cargo.toml

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:243cebc6330729b254fa25be36f67568c1658adb3b82d0736bb8b942928c51f3
+size 28739744

data/Image-arm32

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f78fb74dda2c033f3de3b411d1d34897b3c62c1aa84f4790a3f64a3d28814a7
+size 123724360

data/Image-arm32le

@@ -1,75 +1,64 @@
 use {
     crate::progress::Progress,
-    indicatif::{ParallelProgressIterator, ProgressIterator},
+    dashmap::DashMap,
+    indicatif::ParallelProgressIterator,
     rayon::iter::{IntoParallelRefIterator, ParallelIterator},
-    std::{collections::HashMap, hash::Hash, mem::size_of},
+    std::{hash::Hash, mem::size_of, num::TryFromIntError},
 };
 
 pub struct Addresses<T> {
     addresses: Vec<T>,
 }
 
-impl<T: Copy + Send + Sync + Default + PartialEq + Eq + Hash> Addresses<T> {
-    fn read_addresses<F: Fn(&[u8]) -> T + Sync + Send>(bytes: &[u8], convert: F) -> Vec<T> {
+impl<
+        T: Copy
+            + Send
+            + Sync
+            + Default
+            + PartialEq
+            + Eq
+            + Hash
+            + TryFrom<usize, Error = TryFromIntError>,
+    > Addresses<T>
+{
+    fn get_address_frequencies<F: Fn(&[u8]) -> T + Sync + Send>(
+        bytes: &[u8],
+        convert: F,
+    ) -> DashMap<T, usize> {
         let chunks = bytes.chunks(size_of::<T>()).collect::<Vec<&[u8]>>();
         let pb = Progress::get("Reading addresses", chunks.len());
+        let map = DashMap::<T, usize>::new();
         chunks
             .par_iter()
             .progress_with(pb)
             .map(|&p| convert(p))
             .filter(|&p| p != T::default())
-            .collect::<Vec<T>>()
+            .for_each(|ptr| {
+                *map.entry(ptr).or_insert(0) += 1;
+            });
+        map
     }
 
-    fn get_freqencies(addresses: Vec<T>) -> Vec<HashMap<T, usize>> {
-        /* Calculate frequencies in parallel */
-        let pb = Progress::get("Calculating frequencies", addresses.len());
-        addresses
+    fn get_unique_addresses(frequencies: DashMap<T, usize>) -> Vec<T> {
+        let pb = Progress::get("Finding unique addresses", frequencies.len());
+        frequencies
             .par_iter()
             .progress_with(pb)
-            .fold(HashMap::<T, usize>::new, |mut map, ptr| {
-                if let Some(v) = map.get(ptr) {
-                    map.insert(*ptr, v + 1);
+            .filter_map(|r| {
+                let (&k, &v) = r.pair();
+                if v == 1 {
+                    Some(k)
                 } else {
-                    map.insert(*ptr, 1);
+                    None
                 }
-                map
             })
-            .collect::<Vec<HashMap<T, usize>>>()
-    }
-
-    fn collate_frequencies(frequencies: Vec<HashMap<T, usize>>) -> HashMap<T, usize> {
-        let pb = Progress::get("Collating frequencies", frequencies.len());
-        frequencies.into_iter().progress_with(pb).fold(
-            HashMap::<T, usize>::new(),
-            |mut map, chunk| {
-                for (k, v) in chunk {
-                    if let Some(v) = map.get(&k) {
-                        map.insert(k, v + 1);
-                    } else {
-                        map.insert(k, v);
-                    }
-                }
-                map
-            },
-        )
-    }
-
-    fn get_unique_addresses(frequencies: HashMap<T, usize>) -> Vec<T> {
-        let pb = Progress::get("Finding unique addresses", frequencies.len());
-        frequencies
-            .par_iter()
-            .progress_with(pb)
-            .filter_map(|(k, v)| if *v == 1 { Some(*k) } else { None })
             .collect()
     }
 
     pub fn new<F: Fn(&[u8]) -> T + Sync + Send + Copy>(bytes: &[u8], convert: F) -> Self {
-        let addresses = Self::read_addresses(bytes, convert);
-        println!("Found: {:?} addresses", addresses.len());
-        let frequencies = Self::get_freqencies(addresses);
-        let collated = Self::collate_frequencies(frequencies);
-        let unique = Self::get_unique_addresses(collated);
+        let frequencies = Self::get_address_frequencies(bytes, convert);
+        println!("Found: {:?} addresses", frequencies.len());
+        let unique = Self::get_unique_addresses(frequencies);
         println!("Found: {:?} unique addresses", unique.len());
         Self { addresses: unique }
     }