absolute path support, fixes, documentation improvements, release v.0.3

CHANGELOG.md

@@ -8,10 +8,12 @@ All notable changes to this project will be documented in this file.
 
 ## [0.3.0] - 2021-05-08
 
-- `download_worker_count`
-- Print elapsed time
-- Value Mapping Download
-- Gzip Support not implemented (since most of the data comes as zip)
+- Add: `download_worker_count` config option, and concurrent downloads feature for faster operation
+- Add: Print elapsed time
+- Add: Value Mapping Download
+- Fix: Relative path normalization
+- Add: Using absolute paths for `local_dir` config option.
+- Consider: Gzip Support not implemented (since most of the data comes as zip)
 
 ## [0.2.1] - 2021-03-12
 

Cargo.toml

@@ -23,4 +23,5 @@ regex = "1"
 path-slash = "0.1.4"
 path-absolutize = "3.0.6"
 futures = "0.3"
+remove_dir_all = "0.7"
 # rand = "0.8"

README.md

@@ -35,6 +35,7 @@ It has two main configurations that are documented below:
     }
   },
   "packages": {
+    "local_dir": "relative/path/to/config",
     "filter_rules": [
       { "type": "single", "id": "TrainingPackage" },
       { "type": "single", "id": "eDocumentElectronicInvoicingforYou" },
@@ -114,13 +115,13 @@ OPTIONS:
 
 ### JSON Config File Reference
 
-| Options for Packages Object | Default  | Description                                                                                                                                                                                          |
-| --------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| zip_extraction              | enabled  | Extract artifact contents, this is useful for Git usage. If you prefer to keep artifacts as .zip files for backup, disable this option.                                                              |
-| local_dir                   | ""       | Directory to download artifacts, relative to the config file. By default it is the same as config file directory.                                                                                    |
-| prop_comment_removal        | disabled | Removes auto-generated timestamp comments in `parameters.prop`. Useful for keeping Git history clean. Only works when zip_extraction is enabled. It is disabled by default since it changes content. |
-| filter_rules                | -        | Filter rules to select packages for sync. It can contain simple package id or regex rules.                                                                                                           |
-| download_worker_count       | 5        | Concurrent handling of download per package content and per artifact download. It defaults to 5 workers.                                                                                             |
+| Options for Packages Object | Default  | Description                                                                                                                                                                                                         |
+| --------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| zip_extraction              | enabled  | Extract artifact contents, this is useful for Git usage. If you prefer to keep artifacts as .zip files for backup, disable this option.                                                                             |
+| local_dir                   | "./"     | Directory to download artifacts, it can be relative to the config file or absolute path. By default it is the same directory that contains config file. Regular rules apply for Linux/Windows paths and JSON escape |
+| prop_comment_removal        | disabled | Removes auto-generated timestamp comments in `parameters.prop`. Useful for keeping Git history clean. Only works when zip_extraction is enabled. It is disabled by default since it changes content.                |
+| filter_rules                | -        | Filter rules to select packages for sync. It can contain simple package id or regex rules. Defaults to no package download.                                                                                         |
+| download_worker_count       | 5        | Concurrent handling of download per package content and per artifact download. It defaults to 5 workers.                                                                                                            |
 
 Config file version can be older than tool version(Currently `0.2.0`), this is to prevent unnecessary changes if there are no breaking changes to the config structure.
 

src/lib.rs

@@ -12,7 +12,7 @@ use std::{
     env,
     io::{Read, Write},
     iter::FromIterator,
-    path::PathBuf,
+    path::{Component, Path, PathBuf},
 };
 use std::{fs, io::Cursor, ops::Deref};
 
@@ -262,7 +262,8 @@ async fn process_package(
 > {
     //remove local package contents before download
     let package_dir = data_dir.join(&package_id);
-    let _ = fs::remove_dir_all(package_dir);
+    remove_dir_all::ensure_empty_dir(&package_dir)?;
+    // let _ = fs::remove_dir_all(package_dir);
 
     println!("Processing Package: {:?}", package_id);
 
@@ -474,14 +475,20 @@ pub async fn run_with_config(
         println!("API First Check Successful.");
     }
 
-    let mut data_dir = std::path::PathBuf::from(&config_path);
-    data_dir = data_dir
-        .parent()
-        .unwrap()
-        .canonicalize()
-        .unwrap()
-        .join(&config.packages.local_dir)
-        .to_path_buf();
+    //https://doc.rust-lang.org/std/fs/fn.canonicalize.html
+
+    let normalized_localdir = normalize_path(Path::new(&config.packages.local_dir));
+    let mut data_dir = std::path::PathBuf::from(".");
+    //config path as starting point:
+    data_dir.push(normalize_path(Path::new(&config_path)));
+    data_dir = data_dir.parent().unwrap().to_path_buf();
+
+    //localdir can be relative or absolute
+    data_dir.push(normalized_localdir);
+
+    tokio::fs::create_dir_all(&data_dir).await?;
+    //UNC paths for long windows paths over 260 chars
+    data_dir = data_dir.canonicalize().unwrap();
 
     let api_package_list = get_all_packages(&config, &client, &authorization).await?;
 
@@ -623,3 +630,30 @@ fn basic_auth(user: &str, pass: &str) -> String {
     let authorization = format!("Basic {encoded}", encoded = encoded);
     authorization
 }
+
+fn normalize_path(path: &Path) -> PathBuf {
+    let mut components = path.components().peekable();
+    let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() {
+        components.next();
+        PathBuf::from(c.as_os_str())
+    } else {
+        PathBuf::new()
+    };
+
+    for component in components {
+        match component {
+            Component::Prefix(..) => unreachable!(),
+            Component::RootDir => {
+                ret.push(component.as_os_str());
+            }
+            Component::CurDir => {}
+            Component::ParentDir => {
+                ret.pop();
+            }
+            Component::Normal(c) => {
+                ret.push(c);
+            }
+        }
+    }
+    ret
+}

src/main.rs

@@ -8,7 +8,7 @@ use std::{fs::File, io::Read};
 
 //cli type
 #[derive(Clap, Debug)]
-#[clap(version = "0.3.0-beta", author = "Fatih.Pense @ pizug.com")]
+#[clap(version = "0.3.0", author = "Fatih.Pense @ pizug.com")]
 struct Opts {
     #[clap(short, long, default_value = "./cpi-sync.json")]
     config: String,