fix Leftover

Author: pleasedontbreak123

common/src/utils.rs

@@ -134,6 +134,42 @@ pub fn get_current_bin_name() -> String {
 mod test {
     use super::*;
 
+    #[test]
+    fn test_is_full_hex_object_id() {
+        // Valid SHA-1 (40 hex)
+        assert!(is_full_hex_object_id(&"0".repeat(40)));
+        assert!(is_full_hex_object_id(&"a".repeat(40)));
+        assert!(is_full_hex_object_id(&"A".repeat(40)));
+        assert!(is_full_hex_object_id(&"f".repeat(40)));
+        assert!(is_full_hex_object_id(&"F".repeat(40)));
+
+        // Valid SHA-256 (64 hex)
+        assert!(is_full_hex_object_id(&"0".repeat(64)));
+        let sha256 = "abcdef".repeat(10) + "abcd"; // 60 + 4 = 64
+        assert!(is_full_hex_object_id(&sha256));
+
+        // Invalid lengths (we don't accept short ids)
+        assert!(!is_full_hex_object_id(""));
+        assert!(!is_full_hex_object_id(&"0".repeat(39)));
+        assert!(!is_full_hex_object_id(&"0".repeat(41)));
+        assert!(!is_full_hex_object_id(&"0".repeat(63)));
+        assert!(!is_full_hex_object_id(&"0".repeat(65)));
+
+        // Invalid characters
+        assert!(!is_full_hex_object_id(
+            &(String::from("g") + &"0".repeat(39))
+        ));
+        assert!(!is_full_hex_object_id(
+            &(String::from("-") + &"0".repeat(39))
+        ));
+        assert!(!is_full_hex_object_id(
+            &(String::from(" ") + &"0".repeat(39))
+        ));
+        assert!(!is_full_hex_object_id(
+            &(String::from("é") + &"0".repeat(39))
+        ));
+    }
+
     #[test]
     fn test_check_conventional_commits() {
         // successfull cases

io-orbit/src/adapter.rs

@@ -697,6 +697,18 @@ impl ObjectStoreAdapter {
         Ok(())
     }
 
+    /// Uploads an object using a single `PUT` in **create-only** mode.
+    ///
+    /// This helper is currently used only for **Git objects** (blob/pack data)
+    /// via `put_stream` when `ObjectNamespace::Git` + `UploadStrategy::SinglePut`
+    /// are selected.
+    ///
+    /// Semantics:
+    /// - Uses [`PutMode::Create`], so the backend will fail if the key already exists.
+    /// - This makes writes *idempotent* for content-addressed Git blobs: the first
+    ///   successful upload wins, and later attempts do not silently overwrite data.
+    /// - Callers must ensure that `path` is a content-hash-based key (Git object id),
+    ///   so that "already exists" is expected and safe to ignore at higher layers.
     async fn put_idempotent(
         &self,
         path: &object_store::path::Path,

io-orbit/src/bin/migrate_local_to_s3.rs

@@ -194,6 +194,9 @@ async fn migrate_all(
 
     // List everything under the local root. Passing `None` means "from the root prefix".
     let mut listed = local.list(None);
+    // Bounded collection of join handles to avoid unbounded memory growth when
+    // migrating very large object stores. We cap the number of stored handles
+    // to a small multiple of the concurrency limit.
     let mut tasks = Vec::new();
 
     while let Some(entry) = listed.next().await {
@@ -283,6 +286,16 @@ async fn migrate_all(
 
             Ok(())
         }));
+
+        // Periodically await some tasks to keep the number of in-memory
+        // JoinHandles bounded. Semaphore still enforces the true I/O
+        // concurrency; this only caps bookkeeping overhead.
+        if tasks.len() >= concurrency.saturating_mul(4).max(64) {
+            if let Some(t) = tasks.pop() {
+                t.await
+                    .map_err(|e| MegaError::Other(format!("migration task panicked: {e}")))??;
+            }
+        }
     }
 
     for t in tasks {

jupiter/src/service/mono_service.rs

@@ -70,13 +70,11 @@ impl MonoService {
             .batch_save_model_with_txn(mega_trees, Some(&txn))
             .await?;
         let mega_blobs = converter.mega_blobs.borrow().values().cloned().collect();
+        let raw_blobs = converter.raw_blobs.into_inner();
+        self.git_service.put_objects(raw_blobs).await?;
         self.mono_storage
             .batch_save_model_with_txn(mega_blobs, Some(&txn))
             .await?;
-
-        self.git_service
-            .put_objects(converter.raw_blobs.into_inner())
-            .await?;
         Ok(txn.commit().await?)
     }
 

jupiter/src/storage/mod.rs

@@ -294,31 +294,25 @@ impl Storage {
                 {
                     Ok(blobs) if blobs.is_empty() => {
                         let friendly = format!(
-                            "Blob {hash} not found in both object storage and metadata (likely never written or invalid request)"
+                            "[obj_missing_in_db_and_s3] Blob {hash} not found in both object storage and metadata (likely never written or invalid request)"
                         );
                         tracing::warn!("{}", friendly);
                         MegaError::ObjStorageNotFound(friendly)
                     }
                     Ok(mut blobs) => {
-                        if let Some(blob) = blobs.pop() {
-                            tracing::warn!(
-                                "Object { } missing in S3 but metadata exists; possible data loss or misconfiguration",
-                                blob.blob_id
-                            );
-                            MegaError::ObjStorageInconsistent(format!(
-                                "Object{hash} missing in S3 but metadata exists; possible data loss or misconfiguration "
-                            ))
-                        } else {
-                            tracing::warn!(
-                                "Object missing in S3 but metadata lookup returned unexpected empty result",
-                            );
-                            MegaError::ObjStorageInconsistent(format!(
-                                "Object {hash} missing in S3 but metadata lookup returned unexpected empty result "
-                            ))
-                        }
+                        let blob = blobs.pop().expect(
+                            "blobs is guaranteed non-empty here due to match guard on previous arm",
+                        );
+                        tracing::error!(
+                            "[obj_missing_in_s3_but_has_meta] Object with hash {hash} missing in S3 but metadata exists in DB for blob_id {}; possible data loss or misconfiguration",
+                            blob.blob_id,
+                        );
+                        MegaError::ObjStorageInconsistent(format!(
+                            "[obj_missing_in_s3_but_has_meta] Object {hash} missing in S3 but metadata exists; possible data loss or misconfiguration"
+                        ))
                     }
                     Err(_) => MegaError::ObjStorageInconsistent(format!(
-                        "Failed to query blob {hash} metadata while handling ObjStorageNotFound "
+                        "[obj_meta_lookup_failed] Failed to query blob {hash} metadata while handling ObjStorageNotFound"
                     )),
                 }
             }

mono/src/api/api_router.rs

@@ -7,6 +7,7 @@ use axum::{
     routing::get,
 };
 use ceres::{api_service::ApiHandler, model::git::TreeQuery};
+use common::errors::MegaError;
 use utoipa_axum::{router::OpenApiRouter, routes};
 
 use crate::{
@@ -77,7 +78,10 @@ pub async fn get_blob_file(
             .header("Content-Disposition", file_name)
             .body(Body::from(data))
             .unwrap()),
-        Err(e) => Err(ApiError::not_found(anyhow!("error={}", e))),
+        Err(e) => match e {
+            MegaError::ObjStorageNotFound(_) => Err(ApiError::not_found(anyhow!("error={}", e))),
+            _ => Err(ApiError::internal(anyhow!("error={}", e))),
+        },
     }
 }