From e8643cab9a1b11f25e8806f9f808ef9666c69263 Mon Sep 17 00:00:00 2001 From: Max Truxa Date: Tue, 31 May 2022 10:24:39 +0000 Subject: [PATCH] Implement repo import for prepackged CRS packages --- src/bpt/cli/cmd/repo_import.cpp | 59 +++++++++-------- src/bpt/crs/error.hpp | 4 +- src/bpt/crs/repo.cpp | 109 ++++++++++++++++++++------------ src/bpt/crs/repo.hpp | 4 +- 4 files changed, 107 insertions(+), 69 deletions(-) diff --git a/src/bpt/cli/cmd/repo_import.cpp b/src/bpt/cli/cmd/repo_import.cpp index 3c3ed5e9..bcd85934 100644 --- a/src/bpt/cli/cmd/repo_import.cpp +++ b/src/bpt/cli/cmd/repo_import.cpp @@ -23,7 +23,14 @@ namespace bpt::cli::cmd { namespace { -void _import_file(bpt::crs::repository& repo, path_ref dirpath) { repo.import_dir(dirpath); } +void _import_file(bpt::crs::repository& repo, path_ref pkgpath) { + auto abs_path = fs::canonical(pkgpath); + if (fs::is_regular_file(abs_path)) { + repo.import_targz(pkgpath); + } else { + repo.import_dir(pkgpath); + } +} int _repo_import(const options& opts) { auto repo = bpt::crs::repository::open_existing(opts.repo.repo_dir); @@ -69,22 +76,22 @@ int repo_import(const options& opts) { return bpt_leaf_try { return _repo_import(opts); } bpt_leaf_catch(bpt::crs::e_repo_import_pkg_already_present, bpt::crs::e_repo_importing_package meta, - bpt::crs::e_repo_importing_dir from_dir) { + bpt::crs::e_repo_importing_path from_path) { bpt_log( error, "Refusing to overwrite existing package .br.yellow[{}] (Importing from [.br.yellow[{}]])"_styled, meta.value.id.to_string(), - from_dir.value.string()); + from_path.value.string()); write_error_marker("repo-import-pkg-already-exists"); return 1; } - bpt_leaf_catch(bpt::crs::e_repo_importing_dir crs_dir, - std::error_code ec, + bpt_leaf_catch(bpt::crs::e_repo_importing_path crs_path, + std::error_code ec, boost::leaf::match>, e_sqlite3_error const* sqlite_error) { bpt_log(error, "SQLite error while importing [.br.yellow[{}]]"_styled, - crs_dir.value.string()); + crs_path.value.string()); bpt_log(error, " .br.red[{}]"_styled, ec.message()); if (sqlite_error) { bpt_log(error, " .br.red[{}]"_styled, sqlite_error->value); @@ -93,10 +100,10 @@ int repo_import(const options& opts) { write_error_marker("repo-import-db-error"); return 1; } - bpt_leaf_catch(bpt::e_json_parse_error parse_error, - bpt::crs::e_repo_importing_dir crs_dir, - bpt::crs::e_pkg_json_path pkg_json_path) { - bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, crs_dir.value.string()); + bpt_leaf_catch(bpt::e_json_parse_error parse_error, + bpt::crs::e_repo_importing_path crs_path, + bpt::crs::e_pkg_json_path pkg_json_path) { + bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, crs_path.value.string()); bpt_log(error, " JSON parse error in [.br.yellow[{}]]:"_styled, pkg_json_path.value.string()); @@ -104,10 +111,10 @@ int repo_import(const options& opts) { write_error_marker("repo-import-invalid-crs-json-parse-error"); return 1; } - bpt_leaf_catch(bpt::crs::e_invalid_meta_data error, - bpt::crs::e_repo_importing_dir crs_dir, - bpt::crs::e_pkg_json_path pkg_json_path) { - bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, crs_dir.value.string()); + bpt_leaf_catch(bpt::crs::e_invalid_meta_data error, + bpt::crs::e_repo_importing_path crs_path, + bpt::crs::e_pkg_json_path pkg_json_path) { + bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, crs_path.value.string()); bpt_log(error, "CRS data in [.br.yellow[{}]] is invalid: .br.red[{}]"_styled, pkg_json_path.value.string(), @@ -115,11 +122,11 @@ int repo_import(const options& opts) { write_error_marker("repo-import-invalid-crs-json"); return 1; } - bpt_leaf_catch(bpt::crs::e_invalid_meta_data error, - bpt::crs::e_repo_importing_dir proj_dir, + bpt_leaf_catch(bpt::crs::e_invalid_meta_data error, + bpt::crs::e_repo_importing_path proj_path, bpt::e_open_project, bpt::e_parse_project_manifest_path proj_json_path) { - bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, proj_dir.value.string()); + bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, proj_path.value.string()); bpt_log(error, "Project data in [.br.yellow[{}]] is invalid: .br.red[{}]"_styled, proj_json_path.value.string(), @@ -128,19 +135,19 @@ int repo_import(const options& opts) { return 1; } bpt_leaf_catch(user_error const& exc, - crs::e_repo_importing_dir crs_dir) { + crs::e_repo_importing_path crs_path) { bpt_log(error, "Error while importing [.br.yellow[{}]]: .br.red[{}]"_styled, - crs_dir.value.string(), + crs_path.value.string(), exc.what()); write_error_marker("repo-import-noent"); return 1; } - bpt_leaf_catch(bpt::crs::e_repo_importing_dir crs_dir, - bpt::e_read_file_path const* read_file, - bpt::e_copy_file const* copy_file, - std::error_code ec) { - bpt_log(error, "Error while importing [.br.red[{}]]:"_styled, crs_dir.value.string()); + bpt_leaf_catch(bpt::crs::e_repo_importing_path crs_path, + bpt::e_read_file_path const* read_file, + bpt::e_copy_file const* copy_file, + std::error_code ec) { + bpt_log(error, "Error while importing [.br.red[{}]]:"_styled, crs_path.value.string()); if (read_file) { bpt_log(error, " Error reading file [.br.yellow[{}]]:"_styled, @@ -156,13 +163,13 @@ int repo_import(const options& opts) { write_error_marker("repo-import-noent"); return 1; } - bpt_leaf_catch(crs::e_repo_importing_dir crs_dir, + bpt_leaf_catch(crs::e_repo_importing_path crs_path, crs::e_repo_importing_package meta, crs::e_repo_import_invalid_pkg_version err) { bpt_log(info, "Error while importing .br.yellow[{}] (from [.br.yellow[{}]]):"_styled, meta.value.id.to_string(), - crs_dir.value.string()); + crs_path.value.string()); bpt_log(error, "Invalid 'pkg-version' on package: .br.red[{}]"_styled, err.value); write_error_marker("repo-import-invalid-pkg-version"); return 1; diff --git a/src/bpt/crs/error.hpp b/src/bpt/crs/error.hpp index 8042bd78..b555e3a3 100644 --- a/src/bpt/crs/error.hpp +++ b/src/bpt/crs/error.hpp @@ -12,7 +12,7 @@ struct e_repo_open_path { std::filesystem::path value; }; -struct e_repo_importing_dir { +struct e_repo_importing_path { std::filesystem::path value; }; @@ -32,4 +32,4 @@ struct e_sync_remote { neo::url value; }; -} // namespace bpt::crs \ No newline at end of file +} // namespace bpt::crs diff --git a/src/bpt/crs/repo.cpp b/src/bpt/crs/repo.cpp index 0e83f297..30e6a40e 100644 --- a/src/bpt/crs/repo.cpp +++ b/src/bpt/crs/repo.cpp @@ -27,6 +27,8 @@ #include #include +#include + using namespace bpt; using namespace bpt::crs; using namespace neo::sqlite3::literals; @@ -87,6 +89,17 @@ void archive_package_libraries(path_ref from_dir, path_ref dest_root, const pack } } +void expand_tgz(path_ref tgz_path, path_ref into) { + auto infile = bpt::open_file(tgz_path, std::ios::binary | std::ios::in); + fs::create_directories(into); + neo::expand_directory_targz( + neo::expand_options{ + .destination_directory = into, + .input_name = tgz_path.string(), + }, + infile); +} + } // namespace void repository::_vacuum_and_compress() const { @@ -97,6 +110,51 @@ void repository::_vacuum_and_compress() const { bpt::compress_file_gz(_dirpath / "repo.db", _dirpath / "repo.db.gz").value(); } +package_info repository::_import_dir(const std::filesystem::path& dirpath) { + auto sd = sdist::from_directory(dirpath); + auto& pkg = sd.pkg; + BPT_E_SCOPE(e_repo_importing_package{pkg}); + if (pkg.id.revision < 1) { + BOOST_LEAF_THROW_EXCEPTION(e_repo_import_invalid_pkg_version{ + "Package pkg-version must be a positive non-zero integer in order to be imported into " + "a repository"}); + } + + // Copy the package into a temporary directory + fs::create_directories(tmp_dir()); + auto prep_dir = bpt::temporary_dir::create_in(tmp_dir()); + archive_package_libraries(dirpath, prep_dir.path(), pkg); + fs::create_directories(prep_dir.path()); + bpt::write_file(prep_dir.path() / "pkg.json", pkg.to_json(2)); + + auto tmp_tgz = tmp_dir() / (prep_dir.path().filename().string() + ".tgz"); + neo::compress_directory_targz(prep_dir.path(), tmp_tgz); + neo_defer { std::ignore = ensure_absent(tmp_tgz); }; + + neo::sqlite3::transaction_guard tr{_db.sqlite3_db()}; + bpt_leaf_try { + db_exec( // + _prepare(R"( + INSERT INTO crs_repo_packages (meta_json) + VALUES (?) + )"_sql), + std::string_view(pkg.to_json())) + .value(); + } + bpt_leaf_catch(matchv) { + BOOST_LEAF_THROW_EXCEPTION(current_error(), e_repo_import_pkg_already_present{}); + }; + + auto dest_dir = subdir_of(pkg); + fs::create_directories(dest_dir); + move_file(tmp_tgz, dest_dir / "pkg.tgz").value(); + bpt::copy_file(prep_dir.path() / "pkg.json", dest_dir / "pkg.json").value(); + tr.commit(); + _vacuum_and_compress(); + + return pkg; +} + repository repository::create(path_ref dirpath, std::string_view name) { BPT_E_SCOPE(e_repo_open_path{dirpath}); std::filesystem::create_directories(dirpath); @@ -135,48 +193,19 @@ fs::path repository::subdir_of(const package_info& pkg) const noexcept { / neo::ufmt("{}~{}", pkg.id.version.to_string(), pkg.id.revision); } -void repository::import_dir(path_ref dirpath) { - BPT_E_SCOPE(e_repo_importing_dir{dirpath}); - auto sd = sdist::from_directory(dirpath); - auto& pkg = sd.pkg; - BPT_E_SCOPE(e_repo_importing_package{pkg}); - auto dest_dir = subdir_of(pkg); - fs::create_directories(dest_dir); +void repository::import_targz(path_ref tgz_path) { + BPT_E_SCOPE(e_repo_importing_path{tgz_path}); + fs::create_directories(tmp_dir()); + auto tmp_pkg_dir = bpt::temporary_dir::create_in(tmp_dir()); + expand_tgz(tgz_path, tmp_pkg_dir.path()); + auto pkg = _import_dir(tmp_pkg_dir.path()); - // Copy the package into a temporary directory - auto prep_dir = bpt::temporary_dir::create_in(dest_dir); - archive_package_libraries(dirpath, prep_dir.path(), pkg); - fs::create_directories(prep_dir.path()); - bpt::write_file(prep_dir.path() / "pkg.json", pkg.to_json(2)); - - auto tmp_tgz = pkg_dir() / (prep_dir.path().filename().string() + ".tgz"); - neo::compress_directory_targz(prep_dir.path(), tmp_tgz); - neo_defer { std::ignore = ensure_absent(tmp_tgz); }; - - if (pkg.id.revision < 1) { - BOOST_LEAF_THROW_EXCEPTION(e_repo_import_invalid_pkg_version{ - "Package pkg-version must be a positive non-zero integer in order to be imported into " - "a repository"}); - } - - neo::sqlite3::transaction_guard tr{_db.sqlite3_db()}; - bpt_leaf_try { - db_exec( // - _prepare(R"( - INSERT INTO crs_repo_packages (meta_json) - VALUES (?) - )"_sql), - std::string_view(pkg.to_json())) - .value(); - } - bpt_leaf_catch(matchv) { - BOOST_LEAF_THROW_EXCEPTION(current_error(), e_repo_import_pkg_already_present{}); - }; + NEO_EMIT(ev_repo_imported_package{*this, tgz_path, pkg}); +} - move_file(tmp_tgz, dest_dir / "pkg.tgz").value(); - bpt::copy_file(prep_dir.path() / "pkg.json", dest_dir / "pkg.json").value(); - tr.commit(); - _vacuum_and_compress(); +void repository::import_dir(path_ref dirpath) { + BPT_E_SCOPE(e_repo_importing_path{dirpath}); + auto pkg = _import_dir(dirpath); NEO_EMIT(ev_repo_imported_package{*this, dirpath, pkg}); } diff --git a/src/bpt/crs/repo.hpp b/src/bpt/crs/repo.hpp index a9ed3a4b..0e0239ba 100644 --- a/src/bpt/crs/repo.hpp +++ b/src/bpt/crs/repo.hpp @@ -23,6 +23,7 @@ class repository { , _dirpath(dirpath) {} void _vacuum_and_compress() const; + package_info _import_dir(const std::filesystem::path& dirpath); public: static repository create(const std::filesystem::path& directory, std::string_view name); @@ -31,6 +32,7 @@ class repository { std::filesystem::path subdir_of(const package_info&) const noexcept; auto pkg_dir() const noexcept { return _dirpath / "pkg"; } + auto tmp_dir() const noexcept { return _dirpath / "tmp"; } auto& root() const noexcept { return _dirpath; } std::string name() const; @@ -49,4 +51,4 @@ struct ev_repo_imported_package { package_info const& pkg_meta; }; -} // namespace bpt::crs \ No newline at end of file +} // namespace bpt::crs