address all ISystem comments - nuke some stuff, make it compile but encounter issues at runtime

AnastaZIuk · AnastaZIuk · commit ba669f713836 · 2023-05-19T13:59:49.000+02:00
diff --git a/examples_tests b/examples_tests
@@ -1 +1 @@
-Subproject commit 44cccb47d99e7e5a8cace7e048e8496f6c21c943
+Subproject commit 27347188231e10928268e943e7905b855794b0e3
diff --git a/include/nbl/builtin/common.h b/include/nbl/builtin/common.h
diff --git a/include/nbl/scene/ICullingLoDSelectionSystem.h b/include/nbl/scene/ICullingLoDSelectionSystem.h
@@ -490,7 +490,17 @@ class ICullingLoDSelectionSystem : public virtual core::IReferenceCounted
 			auto getShader = [device]<core::StringLiteral Path>() -> shader_source_and_path
 			{
 				auto system = device->getPhysicalDevice()->getSystem();
-				auto glslFile = system->loadBuiltinData<Path>();
+
+				auto loadBuiltinData = [&](const std::string _path) -> core::smart_refctd_ptr<const nbl::system::IFile>
+				{
+					nbl::system::ISystem::future_t<core::smart_refctd_ptr<nbl::system::IFile>> future;
+					system->createFile(future, system::path(_path), core::bitflag(nbl::system::IFileBase::ECF_READ) | nbl::system::IFileBase::ECF_MAPPABLE);
+					if (future.wait())
+						return future.copy();
+					return nullptr;
+				};
+
+				auto glslFile = loadBuiltinData(Path.value);
 				core::smart_refctd_ptr<asset::ICPUBuffer> glsl;
 				{
 					glsl = core::make_smart_refctd_ptr<asset::ICPUBuffer>(glslFile->getSize());
diff --git a/include/nbl/scene/ISkinInstanceCacheManager.h b/include/nbl/scene/ISkinInstanceCacheManager.h
@@ -26,7 +26,16 @@ class ISkinInstanceCacheManager : public virtual core::IReferenceCounted
 			auto system = device->getPhysicalDevice()->getSystem();
 			auto createShader = [&system,&device](auto uniqueString, asset::IShader::E_SHADER_STAGE type=asset::IShader::ESS_COMPUTE) -> core::smart_refctd_ptr<video::IGPUSpecializedShader>
 			{
-				auto glslFile = system->loadBuiltinData<decltype(uniqueString)>();
+				auto loadBuiltinData = [&](const std::string _path) -> core::smart_refctd_ptr<const nbl::system::IFile>
+				{
+					nbl::system::ISystem::future_t<core::smart_refctd_ptr<nbl::system::IFile>> future;
+					system->createFile(future, system::path(_path), core::bitflag(nbl::system::IFileBase::ECF_READ) | nbl::system::IFileBase::ECF_MAPPABLE);
+					if (future.wait())
+						return future.copy();
+					return nullptr;
+				};
+
+				auto glslFile = loadBuiltinData(uniqueString);
 				core::smart_refctd_ptr<asset::ICPUBuffer> glsl;
 				{
 					glsl = core::make_smart_refctd_ptr<asset::ICPUBuffer>(glslFile->getSize());
@@ -36,9 +45,9 @@ class ISkinInstanceCacheManager : public virtual core::IReferenceCounted
 				return device->createSpecializedShader(shader.get(),{nullptr,nullptr,"main"});
 			};
 
-			auto updateSpec = createShader(NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/glsl/skinning/cache_update.comp")());
-			auto debugDrawVertexSpec = createShader(NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/glsl/skinning/debug.vert")(),asset::IShader::ESS_VERTEX);
-			auto debugDrawFragmentSpec = createShader(NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/material/debug/vertex_normal/specialized_shader.frag")(),asset::IShader::ESS_FRAGMENT);
+			auto updateSpec = createShader("nbl/builtin/glsl/skinning/cache_update.comp");
+			auto debugDrawVertexSpec = createShader("nbl/builtin/glsl/skinning/debug.vert",asset::IShader::ESS_VERTEX);
+			auto debugDrawFragmentSpec = createShader("nbl/builtin/material/debug/vertex_normal/specialized_shader.frag",asset::IShader::ESS_FRAGMENT);
 			if (!updateSpec || !debugDrawVertexSpec || !debugDrawFragmentSpec)
 				return nullptr;
 
diff --git a/include/nbl/scene/ITransformTreeManager.h b/include/nbl/scene/ITransformTreeManager.h
@@ -108,7 +108,16 @@ class ITransformTreeManager : public virtual core::IReferenceCounted
 			auto system = device->getPhysicalDevice()->getSystem();
 			auto createShader = [&system,&device]<core::StringLiteral Path>(asset::IShader::E_SHADER_STAGE type=asset::IShader::ESS_COMPUTE) -> core::smart_refctd_ptr<video::IGPUSpecializedShader>
 			{
-				auto glslFile = system->loadBuiltinData<Path>();
+				auto loadBuiltinData = [&](const std::string _path) -> core::smart_refctd_ptr<const nbl::system::IFile>
+				{
+					nbl::system::ISystem::future_t<core::smart_refctd_ptr<nbl::system::IFile>> future;
+					system->createFile(future, system::path(_path), core::bitflag(nbl::system::IFileBase::ECF_READ) | nbl::system::IFileBase::ECF_MAPPABLE);
+					if (future.wait())
+						return future.copy();
+					return nullptr;
+				};
+
+				auto glslFile = loadBuiltinData(Path.value);
 				core::smart_refctd_ptr<asset::ICPUBuffer> glsl;
 				{
 					glsl = core::make_smart_refctd_ptr<asset::ICPUBuffer>(glslFile->getSize());
diff --git a/include/nbl/system/ISystem.h b/include/nbl/system/ISystem.h
@@ -5,16 +5,15 @@
 #include "nbl/core/declarations.h"
 #include "nbl/core/util/bitflag.h"
 
-//#include "nbl/system/ICancellableAsyncQueueDispatcher.h"
-//#include "nbl/system/IFileArchive.h"
-
-#include "nbl/builtin/common.h"
-
 #include <variant>
 
 #include "nbl/system/IFileArchive.h"
 #include "nbl/system/IAsyncQueueDispatcher.h"
 
+#ifdef _NBL_EMBED_BUILTIN_RESOURCES_
+#include "nbl/builtin/builtinResources.h"
+#endif
+
 namespace nbl::system
 {
 
@@ -61,21 +60,6 @@ class NBL_API2 ISystem : public core::IReferenceCounted
         }
 		#endif
 
-        //! Compile time resource ID
-        template<nbl::core::StringLiteral Path>
-        inline core::smart_refctd_ptr<const IFile> loadBuiltinData()
-        {
-        #ifdef _NBL_EMBED_BUILTIN_RESOURCES_
-            return impl_loadEmbeddedBuiltinData(Path.value,nbl::builtin::get_resource<Path>());
-        #else
-            future_t<core::smart_refctd_ptr<IFile>> future;
-            createFile(future,system::path(Path.value),core::bitflag(IFileBase::ECF_READ)|IFileBase::ECF_MAPPABLE);
-            if (future.wait())
-                return future.copy();
-            return nullptr;
-        #endif
-        }
-
         //
         inline void addArchiveLoader(core::smart_refctd_ptr<IArchiveLoader>&& loader)
         {
@@ -209,9 +193,6 @@ class NBL_API2 ISystem : public core::IReferenceCounted
         explicit ISystem(core::smart_refctd_ptr<ICaller>&& caller);
         virtual ~ISystem() {}
 
-        //
-        core::smart_refctd_ptr<const IFile> impl_loadEmbeddedBuiltinData(const std::string& builtinPath, const std::pair<const uint8_t*,size_t>& found) const;
-
         // given an `absolutePath` find the archive it belongs to
         struct FoundArchiveFile
         {
diff --git a/src/nbl/CMakeLists.txt b/src/nbl/CMakeLists.txt
@@ -116,7 +116,7 @@ nbl_get_conf_dir(NABLA_CONF_DIR_DEBUG Debug)
 nbl_get_conf_dir(NABLA_CONF_DIR_RELEASE Release)
 if(NBL_EMBED_BUILTIN_RESOURCES)	
 	add_subdirectory(builtin)
-	NBL_ADD_BUILTIN_RESOURCES(nblBuiltinResourceData)
+	NBL_ADD_BUILTIN_RESOURCES(nblBuiltinResourceData) # internal, must be added with the macro to properly propagate scope
 endif()
 nbl_get_conf_dir(NABLA_CONF_DIR_RELWITHDEBINFO RelWithDebInfo)
 
diff --git a/src/nbl/asset/IAssetManager.cpp b/src/nbl/asset/IAssetManager.cpp
@@ -242,38 +242,47 @@ void IAssetManager::insertBuiltinAssets()
 		};
 		auto fileSystem = getSystem();
 
-		buildInGLSLShader(fileSystem->loadBuiltinData<NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/specialized_shader/fullscreentriangle.vert")>(),
+        auto loadBuiltinData = [&](const std::string _path) -> core::smart_refctd_ptr<const nbl::system::IFile>
+        {
+            nbl::system::ISystem::future_t<core::smart_refctd_ptr<nbl::system::IFile>> future;
+            fileSystem->createFile(future, system::path(_path), core::bitflag(nbl::system::IFileBase::ECF_READ) | nbl::system::IFileBase::ECF_MAPPABLE);
+            if (future.wait())
+                return future.copy();
+            return nullptr;
+        };
+
+		buildInGLSLShader(loadBuiltinData("nbl/builtin/specialized_shader/fullscreentriangle.vert"),
 			asset::IShader::ESS_VERTEX,
 			{
                 "nbl/builtin/specialized_shader/fullscreentriangle.vert"
             });
-		buildInGLSLShader(fileSystem->loadBuiltinData<NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/material/lambertian/singletexture/specialized_shader.vert")>(),
+		buildInGLSLShader(loadBuiltinData("nbl/builtin/material/lambertian/singletexture/specialized_shader.vert"),
 			asset::IShader::ESS_VERTEX,
 			{
                 "nbl/builtin/material/lambertian/singletexture/specialized_shader.vert",
                 "nbl/builtin/material/debug/vertex_uv/specialized_shader.vert"
             });
-		buildInGLSLShader(fileSystem->loadBuiltinData<NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/material/lambertian/singletexture/specialized_shader.frag")>(), // it somehow adds an extra "tt" raw string to the end of the returned value, beware
+		buildInGLSLShader(loadBuiltinData("nbl/builtin/material/lambertian/singletexture/specialized_shader.frag"), // it somehow adds an extra "tt" raw string to the end of the returned value, beware
 			asset::IShader::ESS_FRAGMENT, 
 			{
                 "nbl/builtin/material/lambertian/singletexture/specialized_shader.frag"
             });
 
-		buildInGLSLShader(fileSystem->loadBuiltinData<NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/material/debug/vertex_normal/specialized_shader.vert")>(),
+		buildInGLSLShader(loadBuiltinData("nbl/builtin/material/debug/vertex_normal/specialized_shader.vert"),
 			asset::IShader::ESS_VERTEX,
 			{
                 "nbl/builtin/material/debug/vertex_normal/specialized_shader.vert"});
-		buildInGLSLShader(fileSystem->loadBuiltinData<NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/material/debug/vertex_color/specialized_shader.vert")>(),
+		buildInGLSLShader(loadBuiltinData("nbl/builtin/material/debug/vertex_color/specialized_shader.vert"),
 			asset::IShader::ESS_VERTEX,
 			{
                 "nbl/builtin/material/debug/vertex_color/specialized_shader.vert"
             });
-		buildInGLSLShader(fileSystem->loadBuiltinData<NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/material/debug/vertex_uv/specialized_shader.frag")>(),
+		buildInGLSLShader(loadBuiltinData("nbl/builtin/material/debug/vertex_uv/specialized_shader.frag"),
 			asset::IShader::ESS_FRAGMENT,
 			{   
                 "nbl/builtin/material/debug/vertex_uv/specialized_shader.frag"
             });
-		buildInGLSLShader(fileSystem->loadBuiltinData<NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/material/debug/vertex_normal/specialized_shader.frag")>(),
+		buildInGLSLShader(loadBuiltinData("nbl/builtin/material/debug/vertex_normal/specialized_shader.frag"),
 			asset::IShader::ESS_FRAGMENT,
 			{
                 "nbl/builtin/material/debug/vertex_normal/specialized_shader.frag",
diff --git a/src/nbl/asset/interchange/CGLTFLoader.cpp b/src/nbl/asset/interchange/CGLTFLoader.cpp
@@ -47,7 +47,18 @@ using namespace nbl::asset;
 		{
 			auto registerShader = [&](auto constexprStringType, IShader::E_SHADER_STAGE stage, const char* extraDefine=nullptr) -> void
 			{
-				core::smart_refctd_ptr<const system::IFile> glslFile = assetManager->getSystem()->loadBuiltinData<decltype(constexprStringType)>();
+				auto fileSystem = assetManager->getSystem();
+
+				auto loadBuiltinData = [&](const std::string _path) -> core::smart_refctd_ptr<const nbl::system::IFile>
+				{
+					nbl::system::ISystem::future_t<core::smart_refctd_ptr<nbl::system::IFile>> future;
+					fileSystem->createFile(future, system::path(_path), core::bitflag(nbl::system::IFileBase::ECF_READ) | nbl::system::IFileBase::ECF_MAPPABLE);
+					if (future.wait())
+						return future.copy();
+					return nullptr;
+				};
+
+				core::smart_refctd_ptr<const system::IFile> glslFile = loadBuiltinData(decltype(constexprStringType)::value);
 				auto glsl = core::make_smart_refctd_ptr<asset::ICPUBuffer>(glslFile->getSize());
 				memcpy(glsl->getPointer(),glslFile->getMappedPointer(),glsl->getSize());
 
diff --git a/src/nbl/asset/interchange/CGraphicsPipelineLoaderMTL.cpp b/src/nbl/asset/interchange/CGraphicsPipelineLoaderMTL.cpp
@@ -30,7 +30,18 @@ CGraphicsPipelineLoaderMTL::CGraphicsPipelineLoaderMTL(IAssetManager* _am, core:
     //create vertex shaders and insert them into cache
     auto registerShader = [&]<core::StringLiteral Path>(ICPUShader::E_SHADER_STAGE stage) -> void
     {
-        core::smart_refctd_ptr<const system::IFile> data = m_assetMgr->getSystem()->loadBuiltinData<Path>();
+        auto fileSystem = m_assetMgr->getSystem();
+
+        auto loadBuiltinData = [&](const std::string _path) -> core::smart_refctd_ptr<const nbl::system::IFile>
+        {
+            nbl::system::ISystem::future_t<core::smart_refctd_ptr<nbl::system::IFile>> future;
+            fileSystem->createFile(future, system::path(_path), core::bitflag(nbl::system::IFileBase::ECF_READ) | nbl::system::IFileBase::ECF_MAPPABLE);
+            if (future.wait())
+                return future.copy();
+            return nullptr;
+        };
+
+        core::smart_refctd_ptr<const system::IFile> data = loadBuiltinData(Path.value);
         auto buffer = core::make_smart_refctd_ptr<asset::ICPUBuffer>(data->getSize()+1u);
         char* bufferPtr = reinterpret_cast<char*>(buffer->getPointer());
         memcpy(bufferPtr, data->getMappedPointer(), data->getSize());
diff --git a/src/nbl/builtin/CMakeLists.txt b/src/nbl/builtin/CMakeLists.txt
@@ -238,7 +238,7 @@ LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "nbl/builtin/glsl/blit/normalizatio
 LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "nbl/builtin/glsl/blit/normalization/descriptors.glsl")
 LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "nbl/builtin/glsl/blit/normalization/shared_normalization.glsl")
 
-macro(NBL_ADD_BUILTIN_RESOURCES _TARGET_) # internal for Nabla only
+macro(NBL_ADD_BUILTIN_RESOURCES _TARGET_) # internal & Nabla only, must be added with the macro to properly propagate scope
 	ADD_CUSTOM_BUILTIN_RESOURCES("${_TARGET_}" NBL_RESOURCES_TO_EMBED "${NBL_ROOT_PATH}/include" "nbl::builtin" "${NBL_ROOT_PATH_BINARY}/include" "${NBL_ROOT_PATH_BINARY}/src")
 endmacro()
 
diff --git a/src/nbl/system/ISystem.cpp b/src/nbl/system/ISystem.cpp
@@ -5,6 +5,7 @@
 
 #include "nbl/system/CArchiveLoaderZip.h"
 #include "nbl/system/CArchiveLoaderTar.h"
+#include "nbl/system/CMountDirectoryArchive.h"
 
 using namespace nbl;
 using namespace nbl::system;
@@ -13,39 +14,18 @@ ISystem::ISystem(core::smart_refctd_ptr<ISystem::ICaller>&& caller) : m_dispatch
 {
     addArchiveLoader(core::make_smart_refctd_ptr<CArchiveLoaderZip>(nullptr));
     addArchiveLoader(core::make_smart_refctd_ptr<CArchiveLoaderTar>(nullptr));
-}
-
-core::smart_refctd_ptr<const IFile> ISystem::impl_loadEmbeddedBuiltinData(const std::string& builtinPath, const std::pair<const uint8_t*,size_t>& found) const
-{
-#ifdef _NBL_EMBED_BUILTIN_RESOURCES_
-    if (found.first && found.second)
-    {
-        auto fileView = core::make_smart_refctd_ptr<CFileView<CNullAllocator>>(
-            builtinPath, 
-            IFile::ECF_READ,
-            const_cast<uint8_t*>(found.first),
-            found.second
-        );
-        return fileView;
-    }
-#endif
-    return nullptr;
+    
+    #ifdef _NBL_EMBED_BUILTIN_RESOURCES_
+    mount(core::make_smart_refctd_ptr<nbl::builtin::CArchive>(nullptr));
+    #else
+    mount(core::make_smart_refctd_ptr<nbl::system::CMountDirectoryArchive>(NBL_BUILTIN_RESOURCES_DIRECTORY_PATH, nullptr), "nbl/builtin");
+    #endif
 }
 
 bool ISystem::exists(const system::path& filename, const core::bitflag<IFile::E_CREATE_FLAGS> flags) const
 {
     const bool writeUsage = flags.value&IFile::ECF_WRITE;
-    if (!writeUsage && builtin::hasPathPrefix(filename.string()))
-    {
-        #ifdef _NBL_EMBED_BUILTIN_RESOURCES_
-            std::pair<const uint8_t*, size_t> found = nbl::builtin::get_resource_runtime(filename.string());
-            if (found.first && found.second)
-                return true;
-        #else
-            if (exists(builtin::getBuiltinResourcesDirectoryPath()/filename,flags))
-                return true;
-        #endif
-    }
+    
     // filename too long
     if (filename.string().size() >= sizeof(SRequestParams_CREATE_FILE::filename))
         return false;
@@ -58,10 +38,6 @@ bool ISystem::exists(const system::path& filename, const core::bitflag<IFile::E_
 
 bool ISystem::isPathReadOnly(const system::path& p) const
 {
-    // first check if its a builtin path
-    if (builtin::hasPathPrefix(p.string()))
-        return true;
-
     // check all parent subpaths
     auto curPath = p;
     while (!curPath.empty() && curPath.parent_path() != curPath)
@@ -104,21 +80,6 @@ core::vector<system::path> ISystem::listItemsInDirectory(const system::path& p)
     }
     else
     {
-        #ifdef _NBL_EMBED_BUILTIN_RESOURCES_
-        // TODO: Change the python generator and make builtinResourceData.cpp's `resourcesByFilename` a nicely named `extern` variable
-        // then in ISystem precompute a [unordered] multimap with keys being builtin subpaths and values being full paths they contain
-        // find all the values here and iterate over them, adding to `res
-        #else
-        err.clear();
-        // check root path is prefixed with "nbl/builtin/"
-        if (builtin::hasPathPrefix(p.string()))
-        {
-            const auto subdirs = std::filesystem::recursive_directory_iterator(builtin::getBuiltinResourcesDirectoryPath()/p,err);
-            if (!err)
-            for (auto entry : subdirs) // there are never any archives inside builtins
-                res.push_back(entry.path());
-        }
-        #endif
         // check for part of subpath being an archive
         auto path = std::filesystem::exists(p) ? std::filesystem::canonical(p):p;
         // going up the directory tree
@@ -215,21 +176,6 @@ void ISystem::createFile(future_t<core::smart_refctd_ptr<IFile>>& future, std::f
     if (std::filesystem::exists(filename))
         filename = std::filesystem::canonical(filename);
 
-    // try builtins
-    if (!(flags.value&IFile::ECF_WRITE) && builtin::hasPathPrefix(filename.string()))
-    {
-        #ifdef _NBL_EMBED_BUILTIN_RESOURCES_
-            auto file = impl_loadEmbeddedBuiltinData(filename.string(),nbl::builtin::get_resource_runtime(filename.string()));
-            if (file)
-            {
-                future.set_result(const_cast<IFile*>(file.get()));
-                return;
-            }
-        #else
-            createFile(future,builtin::getBuiltinResourcesDirectoryPath()/filename,flags);
-            return;
-        #endif
-    }
     // try archives (readonly, for now)
     if (!(flags.value&IFile::ECF_WRITE))
     {
diff --git a/src/nbl/video/utilities/CPropertyPoolHandler.cpp b/src/nbl/video/utilities/CPropertyPoolHandler.cpp
@@ -15,7 +15,16 @@ CPropertyPoolHandler::CPropertyPoolHandler(core::smart_refctd_ptr<ILogicalDevice
 	auto system = m_device->getPhysicalDevice()->getSystem();
 	core::smart_refctd_ptr<asset::ICPUBuffer> glsl;
 	{
-		auto glslFile = system->loadBuiltinData<NBL_CORE_UNIQUE_STRING_LITERAL_TYPE("nbl/builtin/glsl/property_pool/copy.comp")>();
+		auto loadBuiltinData = [&](const std::string _path) -> core::smart_refctd_ptr<const nbl::system::IFile>
+		{
+			nbl::system::ISystem::future_t<core::smart_refctd_ptr<nbl::system::IFile>> future;
+			system->createFile(future, system::path(_path), core::bitflag(nbl::system::IFileBase::ECF_READ) | nbl::system::IFileBase::ECF_MAPPABLE);
+			if (future.wait())
+				return future.copy();
+			return nullptr;
+		};
+
+		auto glslFile = loadBuiltinData("nbl/builtin/glsl/property_pool/copy.comp");
 		glsl = core::make_smart_refctd_ptr<asset::ICPUBuffer>(glslFile->getSize());
 		memcpy(glsl->getPointer(), glslFile->getMappedPointer(), glsl->getSize());
 	}
diff --git a/src/nbl/video/utilities/CScanner.cpp b/src/nbl/video/utilities/CScanner.cpp
