Shader compiler options to save preprocessed and spirv, optimizer upgrade

include/nbl/asset/utils/IShaderCompiler.h

@@ -245,6 +245,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 				@includeFinder Optional parameter; if not nullptr, it will resolve the includes in the code
 				@maxSelfInclusionCount used only when includeFinder is not nullptr
 				@extraDefines adds extra defines to the shader before compilation
+			@optimizerIsExtraPasses Instead of entirely replacing the default optimization passes, run the provided passes after the default ones
+			@preprocessedOutputPath If not empty, the preprocessed shader will be saved to this path
+			@spvOutputPath If not empty, the compiled SPIR-V binary will be saved to this path
 		*/
 		struct SCompilerOptions
 		{
@@ -261,6 +264,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 			SPreprocessorOptions preprocessorOptions = {};
 			CCache* readCache = nullptr;
 			CCache* writeCache = nullptr;
+			bool optimizerIsExtraPasses = false; // Instead of disabling the default opt passes, run the provided optimization passes at the end
+			std::string preprocessedOutputPath = "";
+			std::string spvOutputPath = "";
 		};
 
 		class CCache final : public IReferenceCounted
@@ -269,7 +275,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 
 			public:
 				// Used to check compatibility of Caches before reading
-				constexpr static inline std::string_view VERSION = "1.1.0";
+				constexpr static inline std::string_view VERSION = "1.1.1";
 
 				static auto const SHADER_BUFFER_SIZE_BYTES = sizeof(uint64_t) / sizeof(uint8_t); // It's obviously 8
 
@@ -361,7 +367,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 						public:
 							inline bool operator==(const SCompilerArgs& other) const {
 								bool retVal = true;
-								if (stage != other.stage || targetSpirvVersion != other.targetSpirvVersion || debugInfoFlags != other.debugInfoFlags || preprocessorArgs != other.preprocessorArgs) retVal = false;
+								if (stage != other.stage || targetSpirvVersion != other.targetSpirvVersion || debugInfoFlags != other.debugInfoFlags || preprocessorArgs != other.preprocessorArgs || optimizerIsExtraPasses != other.optimizerIsExtraPasses) retVal = false;
 								if (optimizerPasses.size() != other.optimizerPasses.size()) retVal = false;
 								for (auto passesIt = optimizerPasses.begin(), otherPassesIt = other.optimizerPasses.begin(); passesIt != optimizerPasses.end(); passesIt++, otherPassesIt++) {
 									if (*passesIt != *otherPassesIt) {
@@ -389,6 +395,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 							// Only SEntry should instantiate this struct
 							SCompilerArgs(const SCompilerOptions& options)
 								: stage(options.stage), targetSpirvVersion(options.preprocessorOptions.targetSpirvVersion), debugInfoFlags(options.debugInfoFlags), preprocessorArgs(options.preprocessorOptions)
+								, optimizerIsExtraPasses(options.optimizerIsExtraPasses)
 							{
 								if (options.spirvOptimizer) {
 									for (auto pass : options.spirvOptimizer->getPasses())
@@ -399,6 +406,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 							IShader::E_SHADER_STAGE stage;
 							E_SPIRV_VERSION targetSpirvVersion;
 							std::vector<ISPIRVOptimizer::E_OPTIMIZER_PASS> optimizerPasses;
+							bool optimizerIsExtraPasses;
 							core::bitflag<E_DEBUG_INFO_FLAGS> debugInfoFlags;
 							SPreprocessorArgs preprocessorArgs;
 					};
@@ -458,6 +466,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 
 					bool setContent(const asset::ICPUBuffer* uncompressedSpirvBuffer);
 
+					IShader::E_SHADER_STAGE getShaderStage() const { return compilerArgs.stage; }
+					SPreprocessorArgs getPreprocessorArgs() const { return compilerArgs.preprocessorArgs; }
+
 					core::smart_refctd_ptr<IShader> decompressShader() const;
 
 					// TODO: make some of these private

include/nbl/video/ILogicalDevice.h

@@ -833,6 +833,9 @@ class NBL_API2 ILogicalDevice : public core::IReferenceCounted, public IDeviceMe
             std::span<const asset::IShaderCompiler::SMacroDefinition> extraDefines = {};
             hlsl::ShaderStage stage = hlsl::ShaderStage::ESS_ALL_OR_LIBRARY;
             core::bitflag<asset::IShaderCompiler::E_DEBUG_INFO_FLAGS> debugInfoFlags = asset::IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_NONE;
+            bool optimizerIsExtraPasses = false; // Instead of disabling the default opt passes, run the provided optimization passes at the end
+            std::string preprocessedOutputPath = "";
+            std::string spvOutputPath = "";
         };
         core::smart_refctd_ptr<asset::IShader> compileShader(const SShaderCreationParameters& creationParams);
 

src/nbl/asset/utils/CHLSLCompiler.cpp

@@ -546,6 +546,45 @@ core::smart_refctd_ptr<IShader> CHLSLCompiler::compileToSPIRV_impl(const std::st
     auto newCode = preprocessShader(std::string(code), stage, hlslOptions.preprocessorOptions, dxc_compile_flags, dependencies);
     if (newCode.empty()) return nullptr;
 
+    auto saveToFile = [&](const std::string& filePath, const void* buffer, size_t size, const char* loggerName)
+    {
+        core::smart_refctd_ptr<system::IFile> saveFile;
+
+        system::ISystem::future_t<core::smart_refctd_ptr<system::IFile>> future;
+        // Ensure it doesn't exist
+        m_system->deleteFile(filePath + "_temp");
+        m_system->createFile(future, filePath + "_temp", system::IFile::ECF_WRITE);
+        if (future.wait())
+        {
+            future.acquire().move_into(saveFile);
+            if (saveFile)
+            {
+                system::IFile::success_t succ;
+                saveFile->write(succ, buffer, 0, size);
+                if (!succ)
+                {
+                    logger.log(std::string("Failed Writing To Temp ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
+                    return;
+                }
+                m_system->deleteFile(filePath);
+                auto renameResult = m_system->moveFileOrDirectory(filePath + "_temp", filePath);
+                if (renameResult)
+                {
+                    logger.log(std::string("Failed Saving ") + loggerName + " File. Check it's not open.", nbl::system::ILogger::ELL_ERROR);
+                    // Clean up
+                    m_system->deleteFile(filePath + "_temp");
+                }
+            }
+            else
+                logger.log(std::string("Failed Creating ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
+        }
+        else
+            logger.log(std::string("Failed Creating ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
+    };
+
+    if (!options.preprocessedOutputPath.empty())
+        saveToFile(options.preprocessedOutputPath, newCode.data(), newCode.size(), "Preprocessed Output");
+
     // Suffix is the shader model version
     std::wstring targetProfile(SHADER_MODEL_PROFILE);
 
@@ -584,13 +623,13 @@ core::smart_refctd_ptr<IShader> CHLSLCompiler::compileToSPIRV_impl(const std::st
                 // TODO: add entry point to `CHLSLCompiler::SOptions` and handle it properly in `dxc_compile_flags.empty()`
                 arguments.push_back(L"main");
             }
-            // If a custom SPIR-V optimizer is specified, use that instead of DXC's spirv-opt.
+            // If a custom SPIR-V optimizer is specified and set to replace default optimization passes, use that instead of DXC's spirv-opt.
             // This is how we can get more optimizer options.
             // 
             // Optimization is also delegated to SPIRV-Tools. Right now there are no difference between 
             // optimization levels greater than zero; they will all invoke the same optimization recipe. 
             // https://github.com/Microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#optimization
-            if (hlslOptions.spirvOptimizer)
+            if (hlslOptions.spirvOptimizer && !hlslOptions.optimizerIsExtraPasses)
                 arguments.push_back(L"-O0");
         }
         //
@@ -651,6 +690,9 @@ core::smart_refctd_ptr<IShader> CHLSLCompiler::compileToSPIRV_impl(const std::st
     if (hlslOptions.spirvOptimizer)
         outSpirv = hlslOptions.spirvOptimizer->optimize(outSpirv.get(), logger);
 
+    if (outSpirv && !options.spvOutputPath.empty())
+        saveToFile(options.spvOutputPath, outSpirv->getPointer(), outSpirv->getSize(), "SPIR-V Output");
+
     return core::make_smart_refctd_ptr<asset::IShader>(std::move(outSpirv), IShader::E_CONTENT_TYPE::ECT_SPIRV, hlslOptions.preprocessorOptions.sourceIdentifier.data());
 }
 

src/nbl/asset/utils/IShaderCompiler.cpp

@@ -111,6 +111,7 @@ namespace nbl::system::json {
                 { "shaderStage", shaderStage },
                 { "spirvVersion", spirvVersion },
                 { "optimizerPasses", p.optimizerPasses },
+				{  "optimizerIsExtraPasses", p.optimizerIsExtraPasses },
                 { "debugFlags", debugFlags },
                 { "preprocessorArgs", p.preprocessorArgs },
             };
@@ -122,6 +123,7 @@ namespace nbl::system::json {
             j.at("shaderStage").get_to(shaderStage);
             j.at("spirvVersion").get_to(spirvVersion);
             j.at("optimizerPasses").get_to(p.optimizerPasses);
+			j.at("optimizerIsExtraPasses").get_to(p.optimizerIsExtraPasses);
             j.at("debugFlags").get_to(debugFlags);
             j.at("preprocessorArgs").get_to(p.preprocessorArgs);
             p.stage = static_cast<IShader::E_SHADER_STAGE>(shaderStage);
@@ -502,6 +504,42 @@ core::smart_refctd_ptr<IShader> nbl::asset::IShaderCompiler::compileToSPIRV(cons
 		return IShaderCompiler::writeDepfile(params, dependencies, options.preprocessorOptions.includeFinder, options.preprocessorOptions.logger);
 	};
 
+    auto saveToFile = [&](const std::string& filePath, const void* buffer, size_t size, const char* loggerName)
+        {
+            core::smart_refctd_ptr<system::IFile> saveFile;
+
+            system::ISystem::future_t<core::smart_refctd_ptr<system::IFile>> future;
+            // Ensure it doesn't exist
+            m_system->deleteFile(filePath + "_temp");
+            m_system->createFile(future, filePath + "_temp", system::IFile::ECF_WRITE);
+            if (future.wait())
+            {
+                future.acquire().move_into(saveFile);
+                if (saveFile)
+                {
+                    system::IFile::success_t succ;
+                    saveFile->write(succ, buffer, 0, size);
+                    if (!succ)
+                    {
+                        options.preprocessorOptions.logger.log(std::string("Failed Writing To Temp ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
+                        return;
+                    }
+                    m_system->deleteFile(filePath);
+                    auto renameResult = m_system->moveFileOrDirectory(filePath + "_temp", filePath);
+                    if (renameResult)
+                    {
+                        options.preprocessorOptions.logger.log(std::string("Failed Saving ") + loggerName + " File. Check it's not open.", nbl::system::ILogger::ELL_ERROR);
+                        // Clean up
+                        m_system->deleteFile(filePath + "_temp");
+                    }
+                }
+                else
+                    options.preprocessorOptions.logger.log(std::string("Failed Creating ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
+            }
+            else
+                options.preprocessorOptions.logger.log(std::string("Failed Creating ") + loggerName + " File.", nbl::system::ILogger::ELL_ERROR);
+        };
+
 	CCache::SEntry entry;
 	if (options.readCache || options.writeCache)
 		entry = CCache::SEntry(code, options);
@@ -519,6 +557,22 @@ core::smart_refctd_ptr<IShader> nbl::asset::IShaderCompiler::compileToSPIRV(cons
 			auto shader = found->decompressShader();
 			if (depfileEnabled && !writeDepfileFromDependencies(found->dependencies))
 				return nullptr;
+            if (!options.spvOutputPath.empty())
+                saveToFile(options.spvOutputPath, shader->getContent()->getPointer(), shader->getContent()->getSize(), "SPIR-V");
+            // Entry doesn't store preprocessed shader, so preprocess it
+			if (!options.preprocessedOutputPath.empty())
+            {
+                // copy shader contents 
+                std::string shaderContents = found->mainFileContents;
+                auto stage = options.stage;
+                // Create a copy, cache hit means dependencies haven't changed but there is no function that takes a const pointer. This is meant for debug anyway.
+                auto deps = found->dependencies;
+                auto preprocessedShader = preprocessShader(std::move(shaderContents), stage, options.preprocessorOptions, &deps);
+                if (preprocessedShader.empty())
+                    options.preprocessorOptions.logger.log("Failed to preprocess shader for output.", nbl::system::ILogger::ELL_ERROR);
+                else
+                    saveToFile(options.preprocessedOutputPath, preprocessedShader.data(), preprocessedShader.size(), "Preprocessed Shader");
+            }
 			return shader;
 		}
 	}

src/nbl/system/ISystem.cpp

@@ -125,7 +125,7 @@ bool ISystem::deleteDirectory(const system::path& p)
 bool nbl::system::ISystem::deleteFile(const system::path& p)
 {
     if (std::filesystem::exists(p) && !std::filesystem::is_directory(p))
-        return std::filesystem::remove(p);
+		return std::filesystem::remove(p);
     else
         return false;
 }

src/nbl/video/ILogicalDevice.cpp

@@ -362,11 +362,15 @@ core::smart_refctd_ptr<asset::IShader> ILogicalDevice::compileShader(const SShad
         commonCompileOptions.stage = creationParams.stage;
         commonCompileOptions.debugInfoFlags = creationParams.debugInfoFlags;
         commonCompileOptions.spirvOptimizer = creationParams.optimizer;
+        commonCompileOptions.optimizerIsExtraPasses = creationParams.optimizerIsExtraPasses;
         commonCompileOptions.preprocessorOptions.targetSpirvVersion = m_physicalDevice->getLimits().spirvVersion;
 
         commonCompileOptions.readCache = creationParams.readCache;
         commonCompileOptions.writeCache = creationParams.writeCache;
 
+        commonCompileOptions.preprocessedOutputPath = creationParams.preprocessedOutputPath;
+        commonCompileOptions.spvOutputPath = creationParams.spvOutputPath;
+
         if (sourceContent==asset::IShader::E_CONTENT_TYPE::ECT_HLSL)
         {
             // TODO: add specific HLSLCompiler::SOption params