diff --git a/src/axom/sina/CMakeLists.txt b/src/axom/sina/CMakeLists.txt index c7cb0c0827..b0d6f20ea0 100644 --- a/src/axom/sina/CMakeLists.txt +++ b/src/axom/sina/CMakeLists.txt @@ -45,15 +45,30 @@ set(sina_sources core/Run.cpp ) +# Set CMake Policy +cmake_policy(SET CMP0115 NEW) + # Add Adiak header and source blt_list_append( TO sina_headers ELEMENTS core/AdiakWriter.hpp IF AXOM_USE_ADIAK ) blt_list_append( TO sina_sources ELEMENTS core/AdiakWriter.cpp IF AXOM_USE_ADIAK ) # Add fortran interface for Sina +if(AXOM_USE_HDF5 AND ENABLE_FORTRAN) + set(AXOM_USE_HDF5_FORTRAN ".true.") +else() + set(AXOM_USE_HDF5_FORTRAN ".false.") +endif() + if (ENABLE_FORTRAN) - blt_list_append( TO sina_headers ELEMENTS interface/sina_fortran_interface.h) - blt_list_append( TO sina_sources - ELEMENTS interface/sina_fortran_interface.cpp interface/sina_fortran_interface.f) + blt_list_append(TO sina_headers + ELEMENTS interface/sina_fortran_interface.h) + blt_list_append(TO sina_sources + ELEMENTS interface/sina_fortran_interface.cpp interface/sina_fortran_interface.f) + configure_file(interface/sina_fortran_interface.f.in + interface/sina_fortran_interface.f + @ONLY) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/interface/sina_fortran_interface.f + DESTINATION interface) endif() #------------------------------------------------------------------------------ @@ -87,4 +102,4 @@ endif() if(AXOM_ENABLE_EXAMPLES) add_subdirectory(examples) -endif() +endif() \ No newline at end of file diff --git a/src/axom/sina/core/Document.cpp b/src/axom/sina/core/Document.cpp index 83495e3a1f..dc98593a1c 100644 --- a/src/axom/sina/core/Document.cpp +++ b/src/axom/sina/core/Document.cpp @@ -12,9 +12,18 @@ * ****************************************************************************** */ - #include "axom/sina/core/Document.hpp" +#include "axom/config.hpp" +#include "axom/core/Path.hpp" +#include "axom/core/utilities/StringUtilities.hpp" + +#include "conduit.hpp" +#ifdef AXOM_USE_HDF5 + #include "conduit_relay.hpp" + #include "conduit_relay_io.hpp" +#endif + #include #include #include @@ -22,6 +31,7 @@ #include #include #include +#include namespace axom { @@ -35,6 +45,41 @@ char const RELATIONSHIPS_KEY[] = "relationships"; char const SAVE_TMP_FILE_EXTENSION[] = ".sina.tmp"; } // namespace +void protocolWarn(std::string const protocol, std::string const &name) +{ + std::unordered_map protocolMessages = { + {".json", ".json extension not found, did you mean to save to this format?"}, + {".hdf5", + ".hdf5 extension not found, did you use one of its other supported types? " + "(h5, hdf, ...)"}}; + + Path path(name, '.'); + + if(protocol != '.' + path.baseName()) + { + auto messageIt = protocolMessages.find(protocol); + if(messageIt != protocolMessages.end()) + { + std::cerr << messageIt->second; + } + } +} + +std::string get_supported_file_types() +{ + std::string types = "["; + for(size_t i = 0; i < supported_types.size(); ++i) + { + types += supported_types[i]; + if(i < supported_types.size() - 1) + { + types += ", "; + } + } + types += "]"; + return types; +} + void Document::add(std::unique_ptr record) { records.emplace_back(std::move(record)); @@ -63,63 +108,184 @@ conduit::Node Document::toNode() const return document; } -void Document::createFromNode(conduit::Node const &asNode, - RecordLoader const &recordLoader) +void Document::createFromNode(const conduit::Node &asNode, + const RecordLoader &recordLoader) { - if(asNode.has_child(RECORDS_KEY)) - { - conduit::Node record_nodes = asNode[RECORDS_KEY]; - if(record_nodes.dtype().is_list()) + conduit::Node nodeCopy = asNode; + + auto processChildNodes = [&](const char *key, + std::function addFunc) { + if(nodeCopy.has_child(key)) { - auto recordIter = record_nodes.children(); - while(recordIter.has_next()) + conduit::Node &childNodes = nodeCopy[key]; + + // -- 1. Check if this node is a primitive leaf (throw immediately if so) + // Customize these checks to match exactly what you consider "primitive." + if(childNodes.dtype().is_number() || childNodes.dtype().is_char8_str() || + childNodes.dtype().is_string()) + { + std::ostringstream message; + message << "The '" << key + << "' element of a document cannot be a primitive value."; + throw std::invalid_argument(message.str()); + } + + // -- 2. Not a primitive. Check if it has no children. + if(childNodes.number_of_children() == 0) { - auto record = recordIter.next(); - add(recordLoader.load(record)); + // Turn it into an empty list + childNodes.set(conduit::DataType::list()); + } + + // -- 3. If it's still not a list, throw + if(!childNodes.dtype().is_list()) + { + std::ostringstream message; + message << "The '" << key + << "' element of a document must be an array/list."; + throw std::invalid_argument(message.str()); + } + + // -- 4. Now it's guaranteed to be a list, so iterate + auto childIter = childNodes.children(); + while(childIter.has_next()) + { + conduit::Node child = childIter.next(); + addFunc(child); } } - else + }; + processChildNodes(RECORDS_KEY, [&](conduit::Node &record) { + add(recordLoader.load(record)); + }); + + processChildNodes(RELATIONSHIPS_KEY, [&](conduit::Node &relationship) { + add(Relationship {relationship}); + }); +} + +Document::Document(conduit::Node const &asNode, RecordLoader const &recordLoader) +{ + this->createFromNode(asNode, recordLoader); +} + +Document::Document(std::string const &asJson, RecordLoader const &recordLoader) +{ + conduit::Node asNode; + asNode.parse(asJson, "json"); + this->createFromNode(asNode, recordLoader); +} + +#ifdef AXOM_USE_HDF5 +void removeSlashes(const conduit::Node &originalNode, conduit::Node &modifiedNode) +{ + for(auto it = originalNode.children(); it.has_next();) + { + it.next(); + std::string key = it.name(); + std::string modifiedKey = + axom::utilities::string::replaceAllInstances(key, "/", slashSubstitute); + + modifiedNode[modifiedKey] = it.node(); + + if(it.node().dtype().is_object()) { - std::ostringstream message; - message << "The '" << RECORDS_KEY - << "' element of a document must be an array"; - throw std::invalid_argument(message.str()); + conduit::Node nestedNode; + removeSlashes(it.node(), nestedNode); + modifiedNode[modifiedKey].set(nestedNode); } } +} - if(asNode.has_child(RELATIONSHIPS_KEY)) +void restoreSlashes(const conduit::Node &modifiedNode, conduit::Node &restoredNode) +{ + // Check if List or Object, if its a list the else statement would turn it into an object + // which breaks the Document + + if(modifiedNode.dtype().is_list()) { - conduit::Node relationship_nodes = asNode[RELATIONSHIPS_KEY]; - if(relationship_nodes.dtype().is_list()) + // If its empty with no children it's the end of a tree + + for(auto it = modifiedNode.children(); it.has_next();) { - auto relationshipsIter = relationship_nodes.children(); - while(relationshipsIter.has_next()) + it.next(); + conduit::Node &newChild = restoredNode.append(); + + // Leaves empty nodes empty, if null data is set the + // Document breaks + + if(it.node().dtype().is_string() || it.node().dtype().is_number()) { - auto &relationship = relationshipsIter.next(); - add(Relationship {relationship}); + newChild.set(it.node()); // Lists need .set + } + + // Recursive Call + if(it.node().number_of_children() > 0) + { + restoreSlashes(it.node(), newChild); } } - else + } + else + { + for(auto it = modifiedNode.children(); it.has_next();) { - std::ostringstream message; - message << "The '" << RELATIONSHIPS_KEY - << "' element of a document must be an array"; - throw std::invalid_argument(message.str()); + it.next(); + std::string key = it.name(); + std::string restoredKey = + axom::utilities::string::replaceAllInstances(key, slashSubstitute, "/"); + + // Initialize a new node for the restored key + conduit::Node &newChild = restoredNode.add_child(restoredKey); + + // Leaves empty keys empty but continues recursive call if its a list + if(it.node().dtype().is_string() || it.node().dtype().is_number() || + it.node().dtype().is_object()) + { + newChild.set(it.node()); + } + else if(it.node().dtype().is_list()) + { + restoreSlashes(it.node(), newChild); // Handle nested lists + } + + // If the node has children, recursively restore them + if(it.node().number_of_children() > 0) + { + conduit::Node nestedNode; + restoreSlashes(it.node(), nestedNode); + newChild.set(nestedNode); + } } } } -Document::Document(conduit::Node const &asNode, RecordLoader const &recordLoader) +void Document::toHDF5(const std::string &filename) const { - this->createFromNode(asNode, recordLoader); -} + conduit::Node node; + conduit::Node &recordsNode = node["records"]; + conduit::Node &relationshipsNode = node["relationships"]; -Document::Document(std::string const &asJson, RecordLoader const &recordLoader) -{ - conduit::Node asNode; - asNode.parse(asJson, "json"); - this->createFromNode(asNode, recordLoader); + for(const auto &record : getRecords()) + { + conduit::Node recordNode = record->toNode(); + + removeSlashes(recordNode, recordsNode.append()); + } + + // Process relationships + for(const auto &relationship : getRelationships()) + { + conduit::Node relationshipNode = relationship.toNode(); + + removeSlashes(relationshipNode, relationshipsNode.append()); + } + + conduit::relay::io::save(node, filename, "hdf5"); } +#endif + +// std::string Document::toJson(conduit::index_t indent, conduit::index_t depth, @@ -129,7 +295,9 @@ std::string Document::toJson(conduit::index_t indent, return this->toNode().to_json("json", indent, depth, pad, eoe); } -void saveDocument(Document const &document, std::string const &fileName) +void saveDocument(Document const &document, + std::string const &fileName, + Protocol protocol) { // It is a common use case for users to want to overwrite their files as // the simulation progresses. However, this operation should be atomic so @@ -138,12 +306,33 @@ void saveDocument(Document const &document, std::string const &fileName) // file is in the same directory to ensure that it is part of the same // file system as the destination file so that the move operation is // atomic. + std::string tmpFileName = fileName + SAVE_TMP_FILE_EXTENSION; - auto asJson = document.toJson(); - std::ofstream fout {tmpFileName}; - fout.exceptions(std::ostream::failbit | std::ostream::badbit); - fout << asJson; - fout.close(); + + if(protocol == Protocol::JSON) + { + protocolWarn(".json", fileName); + auto asJson = document.toJson(); + std::ofstream fout {tmpFileName}; + fout.exceptions(std::ostream::failbit | std::ostream::badbit); + fout << asJson; + fout.close(); + } +#ifdef AXOM_USE_HDF5 + else if(protocol == Protocol::HDF5) + { + protocolWarn(".hdf5", fileName); + document.toHDF5(tmpFileName); + } +#endif + else + { + std::ostringstream message; + message << "Invalid format choice. Please choose from one of the supported " + "protocols: " + << get_supported_file_types(); + throw std::invalid_argument(message.str()); + } if(rename(tmpFileName.c_str(), fileName.c_str()) != 0) { @@ -154,20 +343,42 @@ void saveDocument(Document const &document, std::string const &fileName) } } -Document loadDocument(std::string const &path) +Document loadDocument(std::string const &path, Protocol protocol) { - return loadDocument(path, createRecordLoaderWithAllKnownTypes()); + return loadDocument(path, createRecordLoaderWithAllKnownTypes(), protocol); } -Document loadDocument(std::string const &path, RecordLoader const &recordLoader) +Document loadDocument(std::string const &path, + RecordLoader const &recordLoader, + Protocol protocol) { - conduit::Node nodeFromJson; - std::ifstream file_in {path}; + conduit::Node node, modifiedNode; std::ostringstream file_contents; - file_contents << file_in.rdbuf(); - file_in.close(); - nodeFromJson.parse(file_contents.str(), "json"); - return Document {nodeFromJson, recordLoader}; + std::ifstream file_in {path}; + + // Load the file depending on the protocol + switch(protocol) + { + case Protocol::JSON: + file_contents << file_in.rdbuf(); + file_in.close(); + node.parse(file_contents.str(), "json"); + return Document {node, recordLoader}; +#ifdef AXOM_USE_HDF5 + case Protocol::HDF5: + file_in.close(); + conduit::relay::io::load(path, "hdf5", node); + restoreSlashes(node, modifiedNode); + return Document {modifiedNode, recordLoader}; +#endif + default: + std::ostringstream message; + message << "Invalid format choice. Please choose from one of the supported " + "protocols: " + << get_supported_file_types(); + throw std::invalid_argument(message.str()); + break; + } } } // namespace sina diff --git a/src/axom/sina/core/Document.hpp b/src/axom/sina/core/Document.hpp index 04e19c5e7f..d179982448 100644 --- a/src/axom/sina/core/Document.hpp +++ b/src/axom/sina/core/Document.hpp @@ -16,13 +16,14 @@ ****************************************************************************** */ -#include -#include +#include "axom/config.hpp" +#include "axom/sina/core/Record.hpp" +#include "axom/sina/core/Relationship.hpp" #include "conduit.hpp" -#include "axom/sina/core/Record.hpp" -#include "axom/sina/core/Relationship.hpp" +#include +#include #define SINA_FILE_FORMAT_VERSION_MAJOR 1 #define SINA_FILE_FORMAT_VERSION_MINOR 0 @@ -32,12 +33,30 @@ namespace axom namespace sina { +enum class Protocol +{ + JSON, + HDF5 +}; + +const std::vector supported_types = {"JSON", +#ifdef AXOM_USE_HDF5 + "HDF5" +#endif +}; + +/** + * \brief The string used to replace '/' in parent node names when saving to HDF5. + */ +const std::string slashSubstitute = "__SINA_SLASHREPLACE__"; + /** - * \brief An object representing the top-level object of a Sina JSON file + * \brief An object representing the top-level object of a Sina file * - * A Document represents the top-level object of a JSON file conforming to the + * A Document represents the top-level object of a file conforming to the * Sina schema. When serialized, these documents can be ingested into a - * Sina database and used with the Sina tool. + * Sina database and used with the Sina tool. Sina files are defaulted to + * JSON but optionally support HDF5. * * Documents contain at most two objects: a list of Records and a list of Relationships. A simple, empty document: * \code{.json} @@ -75,6 +94,13 @@ namespace sina * \code * axom::sina::saveDocument(myDocument, "path/to/outfile.json") * \endcode + * + * Loading and Saving documents will default to the JSON file type, but if an optional file type is + * loaded the Protocol parameter will control your file type. For example with HDF5: + * \code + * axom::sina::Document myDocument = axom::sina::loadDocument("path/to/infile.hdf5, Protocol::HDF5"); + * axom::sina::saveDocument(myDocument, "path/to/outfile.hdf5", Protocol::HDF5) + * \endcode * * Check the Sina file format version with: * \code @@ -177,6 +203,15 @@ class Document */ conduit::Node toNode() const; +#ifdef AXOM_USE_HDF5 + /** + * \brief Dump this document as an HDF5 File + * + * \param filename the location of which to save the file + */ + void toHDF5(const std::string &filename) const; +#endif + /** * \brief Convert this document to a JSON string. * @@ -187,6 +222,13 @@ class Document const std::string &pad = "", const std::string &eoe = "") const; + /** + * \brief Get the list of file types currently supported by the implementation. + * + * \return a string of supported file types + */ + std::string get_supported_file_types(); + private: /** * Constructor helper method, extracts info from a conduit Node. @@ -200,12 +242,16 @@ class Document /** * \brief Save the given Document to the specified location. If the given file exists, * it will be overwritten. - * + * * \param document the Document to save - * \param fileName the location to which to save the file + * \param fileName the location of which to save the file + * \param protocol the file type requested to save as contained in supported_types, default = JSON * \throws std::ios::failure if there are any IO errors + * std::invalid_argument if the protocol given is an undefined, optional protocol */ -void saveDocument(Document const &document, std::string const &fileName); +void saveDocument(Document const &document, + std::string const &fileName, + Protocol protocol = Protocol::JSON); /** * \brief Get the current file format version. @@ -223,9 +269,11 @@ inline std::string getSinaFileFormatVersion() * knows about will be able to be loaded. * * \param path the file system path from which to load the document + * \param protocol the type of file being loaded, default = JSON * \return the loaded Document */ -Document loadDocument(std::string const &path); +Document loadDocument(std::string const &path, + Protocol protocol = Protocol::JSON); /** * \brief Load a document from the given path. @@ -233,9 +281,13 @@ Document loadDocument(std::string const &path); * \param path the file system path from which to load the document * \param recordLoader the RecordLoader to use to load the different types * of records + * \param protocol the type of file being loaded, default = JSON + * \throws std::invalid_argument if the protocol given is an undefined, optional protocol * \return the loaded Document */ -Document loadDocument(std::string const &path, RecordLoader const &recordLoader); +Document loadDocument(std::string const &path, + RecordLoader const &recordLoader, + Protocol protocol = Protocol::JSON); } // namespace sina } // namespace axom diff --git a/src/axom/sina/docs/imgs/JSON_vs_HDF5_size.png b/src/axom/sina/docs/imgs/JSON_vs_HDF5_size.png new file mode 100644 index 0000000000..c028463e9c Binary files /dev/null and b/src/axom/sina/docs/imgs/JSON_vs_HDF5_size.png differ diff --git a/src/axom/sina/docs/imgs/JSON_vs_HDF5_speed.png b/src/axom/sina/docs/imgs/JSON_vs_HDF5_speed.png new file mode 100644 index 0000000000..b8c8f26448 Binary files /dev/null and b/src/axom/sina/docs/imgs/JSON_vs_HDF5_speed.png differ diff --git a/src/axom/sina/docs/sphinx/documents.rst b/src/axom/sina/docs/sphinx/documents.rst index 64fa8ed775..2a0d270a32 100644 --- a/src/axom/sina/docs/sphinx/documents.rst +++ b/src/axom/sina/docs/sphinx/documents.rst @@ -106,6 +106,30 @@ of the ``Document`` that way: std::cout << myDocument.toJson() << std::endl; } + +------------------------------ +Generating Documents From HDF5 +------------------------------ + +In addition to assembling ``Document`` instances from existing JSON files, it +is possible to generate ``Document`` objects from existing HDF5 files using +conduit. + +Sina's ``saveDocument()`` and ``loadDocument()`` functions support HDF5 assembly if we both +build axom with HDF5 support and provide it with the optional Protocol variable set to HDF5. +If Protocol::HDF5 is attempted while axom does not have HDF5 support, the functions will +recognize they won't have the support for what you are attempting and will return a list of +supported types instead as a runtime error. + +.. code:: cpp + + #include "axom/sina.hpp" + + int main (void) { + axom::sina::Document myDocument = axom::sina::loadDocument("MySinaData.hdf5", axom::sina::Protocol::HDF5); + } + + --------------------------------------------------------- Obtaining Records & Relationships from Existing Documents --------------------------------------------------------- diff --git a/src/axom/sina/docs/sphinx/hdf5_vs_json.rst b/src/axom/sina/docs/sphinx/hdf5_vs_json.rst new file mode 100644 index 0000000000..f33679fb36 --- /dev/null +++ b/src/axom/sina/docs/sphinx/hdf5_vs_json.rst @@ -0,0 +1,55 @@ +.. ## Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and +.. ## other Axom Project Developers. See the top-level LICENSE file for details. +.. ## +.. ## SPDX-License-Identifier: (BSD-3-Clause) + +.. _curvesets-label: + +========== +HDF5 and JSON +========== + +Sina's C++ and Python Code supports loading both JSON and HDF5 files as Documents, +and saving Documents to either file type. + +Both options offer differing strengths and weaknesses when compared to each other, +the following data on the size and speed has been compiled to give a better idea +of which file type fits your requirements. + +========== +Why You Should Use JSON +========== + +* JSON is more universally parsable +* JSON code is human readable allowing you more, easier flexibility with searching +for relevent data. This is most applicable with smaller files since large amounts +of data or curve sets quickly become more efficient to navigate digitally +* JSON outperforms HDF5 in speed and size efficiency when dealing with non-curve set data +and outperforms at smaller curve set sizes (just before :math:`10^2` Curve Sets for Size and +around :math:`10^3.25` for Speed) + +========== +Why You Should Use HDF5 +========== + +* HDF5 offers better size and speed efficiency when dealing with larger files/curve sets +and only outperforms more dramatically as size increases +* Hierarchical structure leads to being 2.5x more size efficient and 5x faster at our largest +tested files + +========== +Images +========== + +JSON vs HDF5 Size +-------- + +.. image:: ../imgs/JSON_vs_HDF5_size.png + :alt: JSON Size / HDF5 Size when dealing with 10 - :math:`10^4` curves with 10 - :math:`10^4` items + + +JSON vs HDF5 Speed +-------- + +.. image:: ../imgs/JSON_vs_HDF5_speed.png + :alt: JSON Size / HDF5 Size when dealing with 10 - :math:`10^4` curves with 10 - :math:`10^4` items \ No newline at end of file diff --git a/src/axom/sina/docs/sphinx/tutorial.rst b/src/axom/sina/docs/sphinx/tutorial.rst index 6be1fd62e9..da9c2af7f1 100644 --- a/src/axom/sina/docs/sphinx/tutorial.rst +++ b/src/axom/sina/docs/sphinx/tutorial.rst @@ -53,7 +53,7 @@ Adding Data Once we have a Record, we can add different types of data to it. Any Datum object that is added will end up in the "data" section of the record in -the JSON file. +the output file. .. literalinclude:: ../../examples/sina_tutorial.cpp :language: cpp @@ -146,9 +146,12 @@ users will be able to search for "temperature" (value = 450), Input and Output ---------------- -Once you have a document, it is easy to save it to a file. After executing -the below, your will output a file named "my_output.json" which you can ingest -into a Sina datastore. +Once you have a document, it is easy to save it to a file. To save to a JSON, we +run the saveDocument() with the optional argument Protocol set to JSON or set as +nothing. Alternatively if you wish to save the document to an HDF5 file: Configure +axom for HDF5 support then you can set saveDocument()'s optional Protocol parameter +to HDF5. After executing the below, you will output a file named "my_output.json" +and a file named "my_output.hdf5", both of which you can ingest into a Sina datastore. .. literalinclude:: ../../examples/sina_tutorial.cpp :language: cpp @@ -157,7 +160,15 @@ into a Sina datastore. If needed, you can also load a document from a file. This can be useful, for example, if you wrote a document when writing a restart and you want to -continue from where you left off. +continue from where you left off. To load from a JSON file simply run loadDocument() +with the optional argument Protocol set to JSON or set as nothing. If you've configured +for, and wish to load from an HDF5 simply set the Protocol to HDF5. + +Note that due to HDF5's handling of '/' as indicators for nested structures, +parent nodes will have '/' changed to the ``slashSubstitute`` variable located in +``axom/sina/core/Document.hpp`` as an HDF5 with saveDocument(). loadDocument() +will restore them to normal: + .. literalinclude:: ../../examples/sina_tutorial.cpp :language: cpp @@ -184,3 +195,4 @@ convert to and from JSON. The user-defined section is exposed as a :language: cpp :start-after: //! [begin user defined] :end-before: //! [end user defined] + diff --git a/src/axom/sina/examples/CMakeLists.txt b/src/axom/sina/examples/CMakeLists.txt index 93e307e99f..f9904282a7 100644 --- a/src/axom/sina/examples/CMakeLists.txt +++ b/src/axom/sina/examples/CMakeLists.txt @@ -27,7 +27,7 @@ set(sina_example_sources sina_view_datum_values.cpp ) -set(sina_example_depends sina conduit slic) +set(sina_example_depends sina conduit::conduit slic) if (ENABLE_FORTRAN) blt_list_append( TO sina_example_sources ELEMENTS sina_fortran.f) @@ -38,16 +38,26 @@ endif() #------------------------------------------------------------------------------ foreach(src ${sina_example_sources}) get_filename_component(exe_name ${src} NAME_WE) + + if (exe_name STREQUAL "sina_fortran") + set(sina_mod_src ${CMAKE_CURRENT_BINARY_DIR}/../interface/sina_fortran_interface.f) + else() + set(sina_mod_src "") + endif() + axom_add_executable( NAME ${exe_name}_ex - SOURCES ${src} + SOURCES ${src} ${sina_mod_src} OUTPUT_DIR ${EXAMPLE_OUTPUT_DIRECTORY} DEPENDS_ON ${sina_example_depends} FOLDER axom/sina/examples - ) + ) + + if (exe_name STREQUAL "sina_fortran") + target_include_directories(${exe_name}_ex PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../interface) + endif() - # Need to add this flag so XL will ignore trailing underscores in fortran function names if (${exe_name}_ex STREQUAL "sina_fortran_ex" AND CMAKE_Fortran_COMPILER_ID STREQUAL "XL") target_compile_options(${exe_name}_ex PRIVATE -qextname) endif() -endforeach() +endforeach() \ No newline at end of file diff --git a/src/axom/sina/examples/sina_basic.cpp b/src/axom/sina/examples/sina_basic.cpp index eba21c0913..4e2e937b61 100644 --- a/src/axom/sina/examples/sina_basic.cpp +++ b/src/axom/sina/examples/sina_basic.cpp @@ -3,6 +3,7 @@ // // SPDX-License-Identifier: (BSD-3-Clause) +#include "axom/config.hpp" #include "axom/sina.hpp" int main(void) @@ -17,5 +18,12 @@ int main(void) // Add the run to the document document.add(std::move(run)); // Save the document directly to a file. - axom::sina::saveDocument(document, "MySinaData.json"); -} \ No newline at end of file + // by specifying Protocol::JSON, we set the file to save as a JSON. + // if we wished, we could provide no third argument since saveDocument defaults to JSON. + axom::sina::saveDocument(document, "MySinaData.json", axom::sina::Protocol::JSON); + +#ifdef AXOM_USE_HDF5 + // by specifying Protocol::HDF5, we also save a copy as an HDF5 file. + axom::sina::saveDocument(document, "MySinaData.hdf5", axom::sina::Protocol::HDF5); +#endif +} diff --git a/src/axom/sina/examples/sina_document_assembly.cpp b/src/axom/sina/examples/sina_document_assembly.cpp index b8ceb44548..6dc4002254 100644 --- a/src/axom/sina/examples/sina_document_assembly.cpp +++ b/src/axom/sina/examples/sina_document_assembly.cpp @@ -3,6 +3,7 @@ // // SPDX-License-Identifier: (BSD-3-Clause) +#include "axom/config.hpp" #include "axom/sina.hpp" int main(void) @@ -32,5 +33,12 @@ int main(void) document.add(relationship); // Save the document directly to a file. + // since we gave saveDocument no optional protocol parameter, it will default to JSON axom::sina::saveDocument(document, "MySinaData.json"); -} \ No newline at end of file + +#ifdef AXOM_USE_HDF5 + // We will also save a copy of the document as an HDF5 file + // which can be done by passing the protocol as HDF5 + axom::sina::saveDocument(document, "MySinaData.hdf5", axom::sina::Protocol::HDF5); +#endif +} diff --git a/src/axom/sina/examples/sina_fortran.f b/src/axom/sina/examples/sina_fortran.f index 8dd2a5e0d4..021e3cd0ac 100644 --- a/src/axom/sina/examples/sina_fortran.f +++ b/src/axom/sina/examples/sina_fortran.f @@ -1,5 +1,6 @@ program example use sina_functions + use hdf5_config implicit none ! data types @@ -28,6 +29,7 @@ program example character(:), allocatable :: tag character(:), allocatable :: units character(20) :: json_fn + character(20) :: hdf5_fn character(15) :: name character(25) :: curve @@ -56,6 +58,9 @@ program example full_path = make_cstring(wrk_dir//''//fle_nme) ofull_path = make_cstring(wrk_dir//''//ofle_nme) json_fn = make_cstring('sina_dump.json') + if (use_hdf5) then + hdf5_fn = make_cstring('sina_dump.hdf5') + end if mime_type = make_cstring('') @@ -149,6 +154,9 @@ program example ! write out the Sina Document print *,'Writing out the Sina Document' call write_sina_document(json_fn) + if (use_hdf5) then + call write_sina_document(hdf5_fn, 1) + end if contains diff --git a/src/axom/sina/examples/sina_tutorial.cpp b/src/axom/sina/examples/sina_tutorial.cpp index e03be73513..92e5ccac11 100644 --- a/src/axom/sina/examples/sina_tutorial.cpp +++ b/src/axom/sina/examples/sina_tutorial.cpp @@ -3,6 +3,7 @@ // // SPDX-License-Identifier: (BSD-3-Clause) +#include "axom/config.hpp" #include "axom/sina.hpp" #include @@ -130,13 +131,20 @@ void gatherAllData(axom::sina::Record &record) void save(axom::sina::Document const &doc) { axom::sina::saveDocument(doc, "my_output.json"); +#ifdef AXOM_USE_HDF5 + axom::sina::saveDocument(doc, "my_output.hdf5", axom::sina::Protocol::HDF5); +#endif } //! [end io write] //! [begin io read] void load() { - axom::sina::Document doc = axom::sina::loadDocument("my_output.json"); + axom::sina::Document doc1 = axom::sina::loadDocument("my_output.json"); +#ifdef AXOM_USE_HDF5 + axom::sina::Document doc2 = + axom::sina::loadDocument("my_output.hdf5", axom::sina::Protocol::HDF5); +#endif } //! [end io read] diff --git a/src/axom/sina/interface/sina_fortran_interface.cpp b/src/axom/sina/interface/sina_fortran_interface.cpp index 6b15d94a93..9ac01a6abb 100644 --- a/src/axom/sina/interface/sina_fortran_interface.cpp +++ b/src/axom/sina/interface/sina_fortran_interface.cpp @@ -258,7 +258,18 @@ extern "C" void sina_add_file_(char *filename, char *mime_type) } } -extern "C" void write_sina_document_(char *input_fn) +extern "C" void write_sina_document_protocol_(char *input_fn, int *protocol) +{ + std::string filename(input_fn); + axom::sina::Protocol proto = static_cast(*protocol); + // Save everything + if(sina_document) + { + axom::sina::saveDocument(*sina_document, filename.c_str(), proto); + } +} + +extern "C" void write_sina_document_noprotocol_(char *input_fn) { std::string filename(input_fn); // Save everything diff --git a/src/axom/sina/interface/sina_fortran_interface.f b/src/axom/sina/interface/sina_fortran_interface.f.in similarity index 74% rename from src/axom/sina/interface/sina_fortran_interface.f rename to src/axom/sina/interface/sina_fortran_interface.f.in index 56fe05ebe6..320c8ada8a 100644 --- a/src/axom/sina/interface/sina_fortran_interface.f +++ b/src/axom/sina/interface/sina_fortran_interface.f.in @@ -1,5 +1,8 @@ module sina_functions + integer, parameter :: JSON = 0 + integer, parameter :: HDF5 = 1 + interface subroutine create_document_and_record(id) @@ -17,14 +20,6 @@ end subroutine sina_add_file end interface - interface - - subroutine write_sina_document(file_nm) - character(*) file_nm - end subroutine write_sina_document - - end interface - interface sina_add subroutine sina_add_long(key, value, units, tags) @@ -114,5 +109,44 @@ subroutine sina_add_curve_long(name, curve, values, n, independent) end subroutine sina_add_curve_long end interface + + interface + + subroutine write_sina_document_protocol(file_nm, protocol) + character(*) file_nm + integer protocol + end subroutine write_sina_document_protocol + + end interface + + interface + + subroutine write_sina_document_noprotocol(file_nm) + character(*) file_nm + end subroutine write_sina_document_noprotocol + + end interface + + + interface write_sina_document + module procedure save_with_protocol + module procedure save_without_protocol + end interface + +contains + subroutine save_with_protocol(fname, proto) + character(*) fname + integer proto + call write_sina_document_protocol(fname, proto) + end subroutine save_with_protocol + subroutine save_without_protocol(fname) + character(*) fname + call write_sina_document_noprotocol(fname) + end subroutine save_without_protocol + +end module -end module \ No newline at end of file +module hdf5_config + implicit none + logical, parameter :: use_hdf5 = @AXOM_USE_HDF5_FORTRAN@ +end module hdf5_config \ No newline at end of file diff --git a/src/axom/sina/interface/sina_fortran_interface.h b/src/axom/sina/interface/sina_fortran_interface.h index 3d06d7d7f9..0a11010c9c 100644 --- a/src/axom/sina/interface/sina_fortran_interface.h +++ b/src/axom/sina/interface/sina_fortran_interface.h @@ -3,9 +3,6 @@ // // SPDX-License-Identifier: (BSD-3-Clause) -#include "axom/sina/core/Document.hpp" -#include "axom/sina/core/Record.hpp" -#include "axom/sina/core/Run.hpp" #include "axom/sina.hpp" extern "C" char *Get_File_Extension(char *); @@ -13,7 +10,8 @@ extern "C" void create_document_and_run_(char *); extern "C" axom::sina::Record *Sina_Get_Run(); extern "C" void sina_add_file_to_record_(char *); extern "C" void sina_add_file_with_mimetype_to_record_(char *, char *); -extern "C" void write_sina_document_(char *); +extern "C" void write_sina_document_protocol_(char *, int *); +extern "C" void write_sina_document_noprotocol_(char *); extern "C" void sina_add_long_(char *, long long int *, char *, char *); extern "C" void sina_add_int_(char *, int *, char *, char *); extern "C" void sina_add_float_(char *, float *, char *, char *); diff --git a/src/axom/sina/tests/CMakeLists.txt b/src/axom/sina/tests/CMakeLists.txt index 109572e3b2..34cb313402 100644 --- a/src/axom/sina/tests/CMakeLists.txt +++ b/src/axom/sina/tests/CMakeLists.txt @@ -110,6 +110,20 @@ if (ENABLE_FORTRAN AND DEFINED PYTHON_EXECUTABLE) COPYONLY ) + # Define Python-friendly boolean for HDF5 guarding in Fortran Test. + if(AXOM_USE_HDF5) + set(AXOM_USE_HDF5_PY "True") + else() + set(AXOM_USE_HDF5_PY "False") + endif() + + # Generate config.py in the test output directory + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/config.py.in + ${TEST_OUTPUT_DIRECTORY}/config.py + @ONLY + ) + axom_add_test( NAME sina_fortran_integration_test COMMAND ${PYTHON_EXECUTABLE} ${TEST_OUTPUT_DIRECTORY}/test_fortran_integration.py -bd ${PROJECT_BINARY_DIR} ) diff --git a/src/axom/sina/tests/config.py.in b/src/axom/sina/tests/config.py.in new file mode 100644 index 0000000000..b6d0a41e78 --- /dev/null +++ b/src/axom/sina/tests/config.py.in @@ -0,0 +1,6 @@ +# Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and +# other Axom Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +AXOM_USE_HDF5 = @AXOM_USE_HDF5_PY@ \ No newline at end of file diff --git a/src/axom/sina/tests/sina_Document.cpp b/src/axom/sina/tests/sina_Document.cpp index 81cf0caa41..74f8278005 100644 --- a/src/axom/sina/tests/sina_Document.cpp +++ b/src/axom/sina/tests/sina_Document.cpp @@ -3,21 +3,26 @@ // // SPDX-License-Identifier: (BSD-3-Clause) +#include "axom/config.hpp" +#include "axom/core/utilities/FileUtilities.hpp" + +#include "axom/sina/core/Document.hpp" +#include "axom/sina/core/Run.hpp" +#include "axom/sina/tests/TestRecord.hpp" + +#include "conduit.hpp" +#include "conduit_relay.hpp" +#include "conduit_relay_io.hpp" + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + #include #include #include #include #include -#include "gtest/gtest.h" -#include "gmock/gmock.h" - -#include "axom/core/utilities/FileUtilities.hpp" -#include "axom/sina/core/Document.hpp" -#include "axom/sina/core/Run.hpp" - -#include "axom/sina/tests/TestRecord.hpp" - namespace axom { namespace sina @@ -34,6 +39,115 @@ char const TEST_RECORD_TYPE[] = "test type"; char const EXPECTED_RECORDS_KEY[] = "records"; char const EXPECTED_RELATIONSHIPS_KEY[] = "relationships"; +// Large JSONs Used For JSON and HDF5 Save Tests +std::string data_json = R"( +{ + "records": [ + { + "type": "run", + "application": "test", + "id": "test_1", + "data": { + "int": { + "value": 500, + "units": "miles" + }, + "str/ings": { + "value": ["z", "o", "o"] + } + }, + "files": { + "test/test.png": {} + } + } + ] +} +)"; + +std::string long_json = R"( +{ + "records": [ + { + "type": "foo", + "id": "test_1", + "user_defined": { + "name": "bob" + }, + "files": { + "foo/bar.png": { + "mimetype": "image" + } + }, + "data": { + "scalar": { + "value": 500, + "units": "miles" + } + } + }, + { + "type": "bar", + "id": "test_2", + "data": { + "scalar_list": { + "value": [1, 2, 3] + }, + "string_list": { + "value": ["a", "wonderful", "world"], + "tags": ["observation"] + } + } + }, + { + "type": "run", + "application": "sina_test", + "id": "test_3", + "data": { + "scalar": { + "value": 12.3, + "units": "g/s", + "tags": ["hi"] + }, + "scalar_list": { + "value": [1, 2, 3.0, 4] + } + } + }, + { + "type": "bar", + "id": "test_4", + "data": { + "string": { + "value": "yarr" + }, + "string_list": { + "value": ["y", "a", "r"] + } + }, + "files": { + "test/test.png": {} + }, + "user_defined": { + "hello": "there" + } + } + ], + "relationships": [ + { + "predicate": "completes", + "subject": "test_2", + "object": "test_1" + }, + { + "subject": "test_3", + "predicate": "overrides", + "object": "test_4" + } + ] +} +)"; + +// Tests TEST(Document, create_fromNode_empty) { conduit::Node documentAsNode; @@ -109,129 +223,6 @@ TEST(Document, create_fromNode_withRelationships) EXPECT_EQ("is related to", relationships[0].getPredicate()); } -TEST(Document, create_fromJson_roundtrip) -{ - std::string orig_json = - "{\"records\": [{\"type\": \"test_rec\",\"id\": " - "\"test\"}],\"relationships\": []}"; - axom::sina::Document myDocument = - Document(orig_json, createRecordLoaderWithAllKnownTypes()); - EXPECT_EQ(0, myDocument.getRelationships().size()); - ASSERT_EQ(1, myDocument.getRecords().size()); - EXPECT_EQ("test_rec", myDocument.getRecords()[0]->getType()); - std::string returned_json = myDocument.toJson(0, 0, "", ""); - EXPECT_EQ(orig_json, returned_json); -} - -TEST(Document, create_fromJson_full) -{ - std::string long_json = R"({ - "records": [ - { - "type": "foo", - "id": "test_1", - "user_defined": { - "name": "bob" - }, - "files": { - "foo/bar.png": { - "mimetype": "image" - } - }, - "data": { - "scalar": { - "value": 500, - "units": "miles" - } - } - }, - { - "type": "bar", - "id": "test_2", - "data": { - "scalar_list": { - "value": [1, 2, 3] - }, - "string_list": { - "value": ["a", "wonderful", "world"], - "tags": ["observation"] - } - } - }, - { - "type": "run", - "application": "sina_test", - "id": "test_3", - "data": { - "scalar": { - "value": 12.3, - "units": "g/s", - "tags": ["hi"] - }, - "scalar_list": { - "value": [1, 2, 3.0, 4] - } - } - }, - { - "type": "bar", - "id": "test_4", - "data": { - "string": { - "value": "yarr" - }, - "string_list": { - "value": ["y", "a", "r"] - } - }, - "files": { - "test/test.png": {} - }, - "user_defined": { - "hello": "there" - } - } - ], - "relationships": [ - { - "predicate": "completes", - "subject": "test_2", - "object": "test_1" - }, - { - "subject": "test_3", - "predicate": "overrides", - "object": "test_4" - } - ] - })"; - axom::sina::Document myDocument = - Document(long_json, createRecordLoaderWithAllKnownTypes()); - EXPECT_EQ(2, myDocument.getRelationships().size()); - auto &records = myDocument.getRecords(); - EXPECT_EQ(4, records.size()); -} - -TEST(Document, create_fromJson_value_check) -{ - std::string data_json = - "{\"records\": [{\"type\": \"run\", \"application\":\"test\", \"id\": " - "\"test_1\",\"data\":{\"int\": {\"value\": 500,\"units\": \"miles\"}, " - "\"str/ings\": {\"value\":[\"z\", \"o\", \"o\"]}}, " - "\"files\":{\"test/test.png\":{}}}]}"; - axom::sina::Document myDocument = - Document(data_json, createRecordLoaderWithAllKnownTypes()); - EXPECT_EQ(0, myDocument.getRelationships().size()); - auto &records = myDocument.getRecords(); - EXPECT_EQ(1, records.size()); - EXPECT_EQ(records[0]->getType(), "run"); - auto &data = records[0]->getData(); - EXPECT_EQ(data.at("int").getScalar(), 500.0); - std::vector expected_string_vals = {"z", "o", "o"}; - EXPECT_EQ(data.at("str/ings").getStringArray(), expected_string_vals); - EXPECT_EQ(records[0]->getFiles().count(File {"test/test.png"}), 1); -} - TEST(Document, toNode_empty) { // A sina document should always have, at minimum, both records and @@ -349,7 +340,45 @@ NamedTempFile::~NamedTempFile() axom::utilities::filesystem::removeFile(fileName.data()); } -TEST(Document, saveDocument) +TEST(Document, create_fromJson_roundtrip_json) +{ + std::string orig_json = + "{\"records\": [{\"type\": \"test_rec\",\"id\": " + "\"test\"}],\"relationships\": []}"; + axom::sina::Document myDocument = + Document(orig_json, createRecordLoaderWithAllKnownTypes()); + EXPECT_EQ(0, myDocument.getRelationships().size()); + ASSERT_EQ(1, myDocument.getRecords().size()); + EXPECT_EQ("test_rec", myDocument.getRecords()[0]->getType()); + std::string returned_json1 = myDocument.toJson(0, 0, "", ""); + EXPECT_EQ(orig_json, returned_json1); +} + +TEST(Document, create_fromJson_full_json) +{ + axom::sina::Document myDocument = + Document(long_json, createRecordLoaderWithAllKnownTypes()); + EXPECT_EQ(2, myDocument.getRelationships().size()); + auto &records1 = myDocument.getRecords(); + EXPECT_EQ(4, records1.size()); +} + +TEST(Document, create_fromJson_value_check_json) +{ + axom::sina::Document myDocument = + Document(data_json, createRecordLoaderWithAllKnownTypes()); + EXPECT_EQ(0, myDocument.getRelationships().size()); + auto &records1 = myDocument.getRecords(); + EXPECT_EQ(1, records1.size()); + EXPECT_EQ(records1[0]->getType(), "run"); + auto &data1 = records1[0]->getData(); + EXPECT_EQ(data1.at("int").getScalar(), 500.0); + std::vector expected_string_vals = {"z", "o", "o"}; + EXPECT_EQ(data1.at("str/ings").getStringArray(), expected_string_vals); + EXPECT_EQ(records1[0]->getFiles().count(File {"test/test.png"}), 1); +} + +TEST(Document, saveDocument_json) { NamedTempFile tmpFile; @@ -433,6 +462,79 @@ TEST(Document, load_defaultRecordLoaders) EXPECT_NE(nullptr, loadedRun); } +#ifdef AXOM_USE_HDF5 +TEST(Document, create_fromJson_roundtrip_hdf5) +{ + std::string orig_json = + "{\"records\": [{\"type\": \"test_rec\",\"id\": " + "\"test\"}],\"relationships\": []}"; + axom::sina::Document myDocument = + Document(orig_json, createRecordLoaderWithAllKnownTypes()); + saveDocument(myDocument, "round_json.hdf5", Protocol::HDF5); + Document loadedDocument = loadDocument("round_json.hdf5", Protocol::HDF5); + EXPECT_EQ(0, loadedDocument.getRelationships().size()); + ASSERT_EQ(1, loadedDocument.getRecords().size()); + EXPECT_EQ("test_rec", loadedDocument.getRecords()[0]->getType()); + std::string returned_json2 = loadedDocument.toJson(0, 0, "", ""); + EXPECT_EQ(orig_json, returned_json2); +} + +TEST(Document, create_fromJson_full_hdf5) +{ + axom::sina::Document myDocument = + Document(long_json, createRecordLoaderWithAllKnownTypes()); + saveDocument(myDocument, "long_json.hdf5", Protocol::HDF5); + Document loadedDocument = loadDocument("long_json.hdf5", Protocol::HDF5); + EXPECT_EQ(2, loadedDocument.getRelationships().size()); + auto &records2 = loadedDocument.getRecords(); + EXPECT_EQ(4, records2.size()); +} + +TEST(Document, create_fromJson_value_check_hdf5) +{ + axom::sina::Document myDocument = + Document(data_json, createRecordLoaderWithAllKnownTypes()); + std::vector expected_string_vals = {"z", "o", "o"}; + saveDocument(myDocument, "data_json.hdf5", Protocol::HDF5); + Document loadedDocument = loadDocument("data_json.hdf5", Protocol::HDF5); + EXPECT_EQ(0, loadedDocument.getRelationships().size()); + auto &records2 = loadedDocument.getRecords(); + EXPECT_EQ(1, records2.size()); + EXPECT_EQ(records2[0]->getType(), "run"); + auto &data2 = records2[0]->getData(); + EXPECT_EQ(data2.at("int").getScalar(), 500.0); + EXPECT_EQ(data2.at("str/ings").getStringArray(), expected_string_vals); + EXPECT_EQ(records2[0]->getFiles().count(File {"test/test.png"}), 1); +} + +TEST(Document, saveDocument_hdf5) +{ + NamedTempFile tmpFile; + + // First, write some random stuff to the temp file to make sure it is + // overwritten. + { + std::ofstream fout {tmpFile.getName()}; + fout << "Initial contents"; + } + + Document document; + document.add( + std::make_unique(ID {"the id", IDType::Global}, "the type")); + + saveDocument(document, tmpFile.getName(), Protocol::HDF5); + + conduit::Node readContents; + conduit::relay::io::load(tmpFile.getName(), "hdf5", readContents); + + ASSERT_TRUE(readContents[EXPECTED_RECORDS_KEY].dtype().is_list()); + EXPECT_EQ(1, readContents[EXPECTED_RECORDS_KEY].number_of_children()); + auto &readRecord = readContents[EXPECTED_RECORDS_KEY][0]; + EXPECT_EQ("the id", readRecord["id"].as_string()); + EXPECT_EQ("the type", readRecord["type"].as_string()); +} +#endif + } // namespace } // namespace testing } // namespace sina diff --git a/src/axom/sina/tests/test_fortran_integration.py b/src/axom/sina/tests/test_fortran_integration.py index 08cb20cf15..a40e102867 100644 --- a/src/axom/sina/tests/test_fortran_integration.py +++ b/src/axom/sina/tests/test_fortran_integration.py @@ -4,6 +4,10 @@ import os import subprocess import unittest +import config + +if (config.AXOM_USE_HDF5): + import h5py def parse_args(): @@ -14,28 +18,33 @@ def parse_args(): # Add other arguments as needed return parser.parse_args() - -class TestFortranExampleIntegration(unittest.TestCase): +# JSON Tests: Will always run +class TestFortranExampleIntegrationJSON(unittest.TestCase): @classmethod def setUpClass(cls): """ - Obtain the binary directory from the CLI. + Obtain the binary directory from the CLI and compile the sina fortran + example needed for these tests if necessary. """ + cwd = os.getcwd() + args = parse_args() cls.binary_dir = args.binary_dir if cls.binary_dir is None: - # Assume we're at /path/to/build_dir/axom/sina/tests so move up to build_dir - cls.binary_dir = os.path.join(os.getcwd(), "..", "..", "..") + # Move up three levels and resolve an absolute path + cls.binary_dir = os.path.abspath(os.path.join(cwd, "../")) + + os.chdir(cls.binary_dir) + + if not os.path.exists(os.path.join(cls.binary_dir, "examples/sina_fortran_ex")): + subprocess.run(["make", "sina_fortran_ex"]) + + os.chdir(cwd) def setUp(self): """ Invoke example Fortran application to dump a sina file """ - sina_fortran_ex_path = os.path.join(self.binary_dir, "examples", "sina_fortran_ex") - if not os.path.exists(sina_fortran_ex_path): - raise FileNotFoundError( - f"The sina_fortran_ex needed for running fortran tests could not be found at path '{sina_fortran_ex_path}'" - ) - subprocess.run([sina_fortran_ex_path]) + subprocess.run([os.path.join(self.binary_dir, "examples/sina_fortran_ex")]) self.dump_file = "sina_dump.json" def tearDown(self): @@ -47,6 +56,7 @@ def test_file_validity(self): try: import jsonschema schema_file = os.path.join(self.binary_dir, "tests", "sina_schema.json") + with io.open(schema_file, "r", encoding="utf-8") as schema: schema = json.load(schema) with io.open(self.dump_file, "r", encoding="utf-8") as loaded_test: @@ -55,7 +65,6 @@ def test_file_validity(self): except ModuleNotFoundError: print("jsonschema module not found. Skipping test_file_validity.") pass - def test_validate_contents_of_record(self): """ Ensure that the record written out matches what we expect """ @@ -69,12 +78,9 @@ def test_validate_contents_of_record(self): self.assertEqual("my_type", record["type"]) # Test the files - path_to_my_file = os.path.join("/path", "to", "my", "file") - my_file = os.path.join(path_to_my_file, "my_file.txt") - other_file = os.path.join(path_to_my_file, "my_other_file.txt") - self.assertEqual(list(record["files"].keys()), [other_file, my_file]) - self.assertEqual(record["files"][other_file]["mimetype"], "png") - self.assertEqual(record["files"][my_file]["mimetype"], "txt") + self.assertEqual(list(record["files"].keys()), ["/path/to/my/file/my_other_file.txt", "/path/to/my/file/my_file.txt"]) + self.assertEqual(record["files"]["/path/to/my/file/my_other_file.txt"]["mimetype"], "png") + self.assertEqual(record["files"]["/path/to/my/file/my_file.txt"]["mimetype"], "txt") # Test the signed variants self.assertEqual("A", record["data"]["char"]["value"]) @@ -116,8 +122,153 @@ def test_validate_contents_of_record(self): self.assertEqual(double_arr, record["curve_sets"][curveset]["dependent"][double_2_name]["value"]) +#HDF5 Test +@unittest.skipUnless(config.AXOM_USE_HDF5, "Requires h5py for HDF5-dependent tests") +class TestFortranExampleIntegrationHDF5(unittest.TestCase): + + @classmethod + def setUpClass(cls): + """ + Obtain the binary directory from the CLI and compile the sina fortran + example needed for these tests if necessary. + """ + cwd = os.getcwd() + + args = parse_args() + cls.binary_dir = args.binary_dir + if cls.binary_dir is None: + # Move up three levels and resolve an absolute path + cls.binary_dir = os.path.abspath(os.path.join(cwd, "../")) + + os.chdir(cls.binary_dir) + + if not os.path.exists(os.path.join(cls.binary_dir, "examples/sina_fortran_ex")): + subprocess.run(["make", "sina_fortran_ex"]) + + os.chdir(cwd) + + def setUp(self): + """ Invoke example Fortran application to dump a sina file """ + subprocess.run([os.path.join(self.binary_dir, "examples/sina_fortran_ex")]) + self.dump_file = "sina_dump.hdf5" + + def tearDown(self): + os.remove(self.dump_file) + + def extract_hdf5_value(self, value): + # If the value is an h5py.Dataset, retrieve its underlying data. + if isinstance(value, h5py.Dataset): + value = value[()] + + # If the value is a bytes instance, decode it. + if isinstance(value, bytes): + return value.decode("utf-8").strip("\0").strip() + + # If the value is a list or tuple of bytes, join them into a single bytes object and decode. + if isinstance(value, (list, tuple)) and value and all(isinstance(item, bytes) for item in value): + joined = b"".join(value) + return joined.decode("utf-8").strip("\0").strip() + + # If the value has a 'tolist' method (e.g., from an array-like object), convert it and process. + if hasattr(value, "tolist"): + converted = value.tolist() + if isinstance(converted, (list, tuple)) and converted and all(isinstance(item, bytes) for item in converted): + joined = b"".join(converted) + return joined.decode("utf-8").strip("\0").strip() + if isinstance(converted, list) and len(converted) == 1: + return converted[0] + return converted + + # Otherwise, return the value as-is. + return value + + def test_validate_contents_of_record(self): + with h5py.File(self.dump_file, "r") as f: + record = f["records"]["0"] + + # Validate metadata + self.assertEqual("my_rec_id", self.extract_hdf5_value(record["id"])) + self.assertEqual("my_type", self.extract_hdf5_value(record["type"])) + + # Validate Files + files_group = record["files"] + expected_file_keys = [ + "__SINA_SLASHREPLACE__path__SINA_SLASHREPLACE__to__SINA_SLASHREPLACE__my__SINA_SLASHREPLACE__file__SINA_SLASHREPLACE__my_other_file.txt", + "__SINA_SLASHREPLACE__path__SINA_SLASHREPLACE__to__SINA_SLASHREPLACE__my__SINA_SLASHREPLACE__file__SINA_SLASHREPLACE__my_file.txt" + ] + self.assertEqual(sorted(list(files_group.keys())), sorted(expected_file_keys)) + self.assertEqual(self.extract_hdf5_value( + files_group["__SINA_SLASHREPLACE__path__SINA_SLASHREPLACE__to__SINA_SLASHREPLACE__my__SINA_SLASHREPLACE__file__SINA_SLASHREPLACE__my_other_file.txt"]["mimetype"]), + "png") + self.assertEqual(self.extract_hdf5_value( + files_group["__SINA_SLASHREPLACE__path__SINA_SLASHREPLACE__to__SINA_SLASHREPLACE__my__SINA_SLASHREPLACE__file__SINA_SLASHREPLACE__my_file.txt"]["mimetype"]), + "txt") + + # Validate Data + data_group = record["data"] + self.assertEqual(self.extract_hdf5_value(data_group["char"]["value"]), "A") + self.assertEqual(self.extract_hdf5_value(data_group["int"]["value"]), 10) + self.assertEqual(self.extract_hdf5_value(data_group["logical"]["value"]), 0) + self.assertEqual(self.extract_hdf5_value(data_group["long"]["value"]), 1000000000.0) + self.assertAlmostEqual(self.extract_hdf5_value(data_group["real"]["value"]), 1.23456704616547) + self.assertAlmostEqual(self.extract_hdf5_value(data_group["double"]["value"]), 0.810000002384186) + + self.assertEqual(self.extract_hdf5_value(data_group["u_char"]["value"]), "A") + self.assertEqual(self.extract_hdf5_value(data_group["u_char"]["units"]), "kg") + self.assertEqual(self.extract_hdf5_value(data_group["u_int"]["value"]), 10) + self.assertEqual(self.extract_hdf5_value(data_group["u_int"]["units"]), "kg") + self.assertEqual(self.extract_hdf5_value(data_group["u_logical"]["value"]), 1.0) + self.assertEqual(self.extract_hdf5_value(data_group["u_logical"]["units"]), "kg") + self.assertEqual(self.extract_hdf5_value(data_group["u_long"]["value"]), 1000000000.0) + self.assertEqual(self.extract_hdf5_value(data_group["u_long"]["units"]), "kg") + self.assertAlmostEqual(self.extract_hdf5_value(data_group["u_real"]["value"]), 1.23456704616547) + self.assertEqual(self.extract_hdf5_value(data_group["u_real"]["units"]), "kg") + self.assertAlmostEqual(self.extract_hdf5_value(data_group["u_double"]["value"]), 0.810000002384186) + self.assertEqual(self.extract_hdf5_value(data_group["u_double"]["units"]), "kg") + self.assertAlmostEqual(self.extract_hdf5_value(data_group["u_double_w_tag"]["value"]), 0.810000002384186) + self.assertEqual(self.extract_hdf5_value(data_group["u_double_w_tag"]["units"]), "kg") + + tags_value = data_group["u_double_w_tag"]["tags"] + if isinstance(tags_value, h5py.Group): + if len(tags_value) == 1: + inner_dataset = list(tags_value.values())[0] + tags_value = self.extract_hdf5_value(inner_dataset) + if isinstance(tags_value, str): + tags_value = [tag.strip() for tag in tags_value.split(',')] + self.assertEqual(tags_value, ["new_fancy_tag"]) + + # Validate Curves + curveset_group = record["curve_sets"]["my_curveset"] + independent_group = curveset_group["independent"] + dependent_group = curveset_group["dependent"] + + nums = list(range(1, 21)) + real_arr = [i for i in nums] + double_arr = [i * 2 for i in nums] + int_arr = [i * 3 for i in nums] + long_arr = [i * 4 for i in nums] + + for kind, grp in (("indep", independent_group), ("dep", dependent_group)): + for val_type, target in (("real", real_arr), + ("double", double_arr), + ("int", int_arr), + ("long", long_arr)): + curve_name = f"my_{kind}_curve_{val_type}" + self.assertIn(curve_name, grp) + curve_val = self.extract_hdf5_value(grp[curve_name]["value"]) + self.assertEqual(curve_val, target) + + double_2_name = "my_dep_curve_double_2" + self.assertIn(double_2_name, dependent_group) + curve_double_2 = self.extract_hdf5_value(dependent_group[double_2_name]["value"]) + self.assertEqual(curve_double_2, double_arr) + + if __name__ == "__main__": # Doing the below instead of unittest.main() so that we can print to stdout - suite = unittest.TestLoader().loadTestsFromTestCase(TestFortranExampleIntegration) + suite = unittest.TestSuite() + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestFortranExampleIntegrationJSON)) + if config.AXOM_USE_HDF5: + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestFortranExampleIntegrationHDF5)) runner = unittest.TextTestRunner(buffer=False) runner.run(suite) \ No newline at end of file