refactor(server): extract data transfer crate (#1243)

Cargo.toml

@@ -32,6 +32,7 @@ members = [
     "crates/adapters/curvine-ufs-opendal",
     "crates/adapters/curvine-ufs-oss-hdfs",
     "crates/adapters/curvine-hdfs-jni",
+    "crates/server/curvine-data-transfer",
     "curvine-master",
     "curvine-worker",
     "curvine-common",
@@ -74,6 +75,7 @@ default-members = [
     "crates/infra/curvine-io",
     "crates/metadata/curvine-raft",
     "crates/adapters/curvine-storage-local",
+    "crates/server/curvine-data-transfer",
     "curvine-master",
     "curvine-worker",
     "curvine-common",
@@ -112,6 +114,7 @@ curvine-storage-spdk = { path = "crates/adapters/curvine-storage-spdk" }
 curvine-ufs-opendal = { path = "crates/adapters/curvine-ufs-opendal" }
 curvine-ufs-oss-hdfs = { path = "crates/adapters/curvine-ufs-oss-hdfs" }
 curvine-hdfs-jni = { path = "crates/adapters/curvine-hdfs-jni" }
+curvine-data-transfer = { path = "crates/server/curvine-data-transfer" }
 curvine-fault = { path = "crates/infra/curvine-fault" }
 curvine-error = { path = "crates/common/curvine-error" }
 curvine-proto = { path = "crates/common/curvine-proto" }

crates/server/curvine-data-transfer/Cargo.toml

@@ -0,0 +1,51 @@
+[package]
+name = "curvine-data-transfer"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[features]
+default = []
+transfer = ["dep:uuid"]
+transfer-web = ["transfer", "dep:axum", "dep:curvine-web"]
+transfer-store-sqlite = ["transfer", "dep:rusqlite"]
+transfer-store-mysql = ["transfer", "dep:mysql"]
+transfer-server = [
+    "transfer-web",
+    "transfer-store-sqlite",
+    "transfer-store-mysql",
+]
+opendal = ["curvine-unified-fs/opendal"]
+opendal-s3 = ["opendal", "curvine-unified-fs/opendal-s3"]
+opendal-oss = ["opendal", "curvine-unified-fs/opendal-oss"]
+opendal-gcs = ["opendal", "curvine-unified-fs/opendal-gcs"]
+opendal-azblob = ["opendal", "curvine-unified-fs/opendal-azblob"]
+opendal-cos = ["opendal", "curvine-unified-fs/opendal-cos"]
+opendal-hdfs = ["opendal", "curvine-unified-fs/opendal-hdfs"]
+opendal-webhdfs = ["opendal", "curvine-unified-fs/opendal-webhdfs"]
+opendal-hdfs-native = ["opendal", "curvine-unified-fs/opendal-hdfs-native"]
+oss-hdfs = ["curvine-unified-fs/oss-hdfs"]
+
+[dependencies]
+orpc = { workspace = true }
+curvine-common = { workspace = true }
+curvine-client-core = { workspace = true }
+curvine-job-client = { workspace = true }
+curvine-unified-fs = { workspace = true }
+curvine-ufs-api = { workspace = true }
+curvine-web = { workspace = true, optional = true }
+prost = { workspace = true }
+bytes = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+log = { workspace = true }
+tokio = { workspace = true }
+futures = { workspace = true }
+dashmap = { workspace = true }
+crossbeam = { workspace = true }
+parking_lot = { workspace = true }
+once_cell = { workspace = true }
+axum = { workspace = true, optional = true }
+rusqlite = { workspace = true, optional = true }
+mysql = { workspace = true, optional = true }
+uuid = { workspace = true, optional = true }

crates/server/curvine-data-transfer/src/common/mod.rs

@@ -0,0 +1,20 @@
+// Copyright 2025 OPPO.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod ufs_client;
+pub mod ufs_manager;
+
+mod ufs_factory;
+pub use self::ufs_client::UfsClient;
+pub use self::ufs_factory::UfsFactory;

curvine-server/src/common/ufs_client.rs → crates/server/curvine-data-transfer/src/common/ufs_client.rs

@@ -14,12 +14,12 @@
 
 #![allow(unused)]
 
-use curvine_client::unified::{UfsFileSystem, UnifiedReader, UnifiedWriter};
 use curvine_common::error::FsError;
 use curvine_common::fs::{CurvineURI, FileSystem, Path};
 use curvine_common::state::FileStatus;
 use curvine_common::FsResult;
 use curvine_ufs_api::fs::ufs_context::UFSContext;
+use curvine_unified_fs::{UfsFileSystem, UnifiedReader, UnifiedWriter};
 use std::sync::Arc;
 
 #[derive(Clone)]

curvine-master/src/common/ufs_factory.rs → crates/server/curvine-data-transfer/src/common/ufs_factory.rs

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::master::JobWorkerClient;
-use curvine_client::unified::{MountValue, UfsFileSystem};
+use crate::JobWorkerClient;
 use curvine_common::conf::ClientConf;
 use curvine_common::state::{MountInfo, WorkerAddress};
 use curvine_common::FsResult;
+use curvine_unified_fs::{MountValue, UfsFileSystem};
 use orpc::client::ClientFactory;
 use orpc::io::net::InetAddr;
 use orpc::runtime::Runtime;

curvine-server/src/common/ufs_manager.rs → crates/server/curvine-data-transfer/src/common/ufs_manager.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::common::UfsClient;
-use curvine_client::file::FsClient;
+use curvine_client_core::file::FsClient;
 use curvine_common::conf::{UfsConf, UfsConfBuilder};
 use curvine_common::fs::{CurvineURI, Path};
 use curvine_common::state::MountInfo;

crates/server/curvine-data-transfer/src/lib.rs

@@ -0,0 +1,24 @@
+// Copyright 2025 OPPO.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod common;
+#[cfg(feature = "transfer")]
+pub mod transfer;
+pub mod worker;
+
+mod job_worker_client;
+pub use self::job_worker_client::JobWorkerClient;
+
+mod rpc_context;
+pub use self::rpc_context::RpcContext;