diff --git a/examples_tests b/examples_tests index 85d44671d1..2581492e56 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 85d44671d137669ce51d973c8cf76b38dad5a12a +Subproject commit 2581492e56a46699a3db9ee3c31bac89a3b1edfb diff --git a/include/nbl/asset/IPipeline.h b/include/nbl/asset/IPipeline.h index dddfe4b528..ada60fee9f 100644 --- a/include/nbl/asset/IPipeline.h +++ b/include/nbl/asset/IPipeline.h @@ -23,13 +23,13 @@ namespace nbl::asset Vulkan supports multiple types of pipelines: - graphics pipeline - compute pipeline - - TODO: Raytracing + - raytracing pipeline */ class IPipelineBase { public: // TODO: tbf these shouldn't even be laying around in `nbl::asset`, we should probably move to `nbl::video::IGPUPipelineBase` - enum class CreationFlags : uint64_t + enum class CreationFlags : uint64_t { NONE = 0, // disallowed in maintanance5 DISABLE_OPTIMIZATIONS = 1 << 0, @@ -47,10 +47,8 @@ class IPipelineBase // This is for NV-raytracing extension. Now this is done via IDeferredOperation //DEFER_COMPILE_NV = 1<<5, - // We use Renderdoc to take care of this for us, - // we won't be parsing the statistics and internal representation ourselves. - //CAPTURE_STATISTICS = 1<<6, - //CAPTURE_INTERNAL_REPRESENTATIONS = 1<<7, + CAPTURE_STATISTICS = 1<<6, + CAPTURE_INTERNAL_REPRESENTATIONS = 1<<7, // Will soon be deprecated due to // https://github.com/Devsh-Graphics-Programming/Nabla/issues/854 diff --git a/include/nbl/asset/IRayTracingPipeline.h b/include/nbl/asset/IRayTracingPipeline.h index 23d2fac81f..fe8dd92de4 100644 --- a/include/nbl/asset/IRayTracingPipeline.h +++ b/include/nbl/asset/IRayTracingPipeline.h @@ -31,6 +31,8 @@ class IRayTracingPipelineBase : public virtual core::IReferenceCounted NO_NULL_MISS_SHADERS = 1<<16, NO_NULL_INTERSECTION_SHADERS = 1<<17, ALLOW_MOTION = 1<<20, + CAPTURE_STATISTICS = base_flag(CAPTURE_STATISTICS), + CAPTURE_INTERNAL_REPRESENTATIONS = base_flag(CAPTURE_INTERNAL_REPRESENTATIONS), }; #undef base_flag diff --git a/include/nbl/video/CVulkanRayTracingPipeline.h b/include/nbl/video/CVulkanRayTracingPipeline.h index a9bc476f43..791d6a766e 100644 --- a/include/nbl/video/CVulkanRayTracingPipeline.h +++ b/include/nbl/video/CVulkanRayTracingPipeline.h @@ -25,6 +25,8 @@ class CVulkanRayTracingPipeline final : public IGPURayTracingPipeline inline VkPipeline getInternalObject() const { return m_vkPipeline; } + void populateExecutableInfo(bool includeInternalRepresentations); + virtual const SShaderGroupHandle& getRaygen() const override; virtual std::span getMissHandles() const override; virtual std::span getHitHandles() const override; diff --git a/include/nbl/video/IGPUComputePipeline.h b/include/nbl/video/IGPUComputePipeline.h index 9854725cd1..deb0545f62 100644 --- a/include/nbl/video/IGPUComputePipeline.h +++ b/include/nbl/video/IGPUComputePipeline.h @@ -38,6 +38,8 @@ class IGPUComputePipeline : public IGPUPipeline stages = hlsl::ShaderStage::ESS_UNKNOWN; + uint32_t subgroupSize = 0; + std::string statistics; + std::string internalRepresentations; + }; + + inline std::span getExecutableInfo() const { return m_executableInfo; } + + protected: + core::vector m_executableInfo; }; // Common Base class for pipelines @@ -146,4 +162,51 @@ class IGPUPipeline : public IBackendObject, public PipelineNonBackendObjectBase, } +namespace nbl::system::impl +{ + +template<> +struct to_string_helper +{ + static std::string __call(const video::IGPUPipelineBase::SExecutableInfo& info) + { + std::string result; + result += "======== "; + result += info.name; + result += " ========\n"; + result += info.description; + result += "\nSubgroup Size: "; + result += std::to_string(info.subgroupSize); + if (!info.statistics.empty()) + { + result += "\n"; + result += info.statistics; + } + if (!info.internalRepresentations.empty()) + { + result += "\n"; + result += info.internalRepresentations; + } + return result; + } +}; + +// Another version for core::vector? +template<> +struct to_string_helper> +{ + static std::string __call(const std::span& infos) + { + std::string result; + for (const auto& info : infos) + { + result += to_string_helper::__call(info); + result += "\n"; + } + return result; + } +}; + +} + #endif diff --git a/include/nbl/video/ILogicalDevice.h b/include/nbl/video/ILogicalDevice.h index ae351fdecd..756b417c79 100644 --- a/include/nbl/video/ILogicalDevice.h +++ b/include/nbl/video/ILogicalDevice.h @@ -14,6 +14,7 @@ #include "nbl/video/IGPUCommandBuffer.h" #include "nbl/video/CThreadSafeQueueAdapter.h" #include "nbl/video/CJITIncludeLoader.h" +#include "nbl/system/to_string.h" #include "git_info.h" #define NBL_LOG_FUNCTION m_logger.log @@ -1273,6 +1274,20 @@ class NBL_API2 ILogicalDevice : public core::IReferenceCounted, public IDeviceMe return {}; } + // CAPTURE_STATISTICS and CAPTURE_INTERNAL_REPRESENTATIONS require pipelineExecutableInfo feature + constexpr auto CaptureStatsFlag = CreationParams::FLAGS::CAPTURE_STATISTICS; + constexpr auto CaptureIRFlag = CreationParams::FLAGS::CAPTURE_INTERNAL_REPRESENTATIONS; + if (ci.getFlags().hasFlags(CaptureStatsFlag) && !getEnabledFeatures().pipelineExecutableInfo) + { + NBL_LOG_ERROR("CAPTURE_STATISTICS flag requires `pipelineExecutableInfo` feature (params[%d])", i); + return {}; + } + if (ci.getFlags().hasFlags(CaptureIRFlag) && !getEnabledFeatures().pipelineExecutableInfo) + { + NBL_LOG_ERROR("CAPTURE_INTERNAL_REPRESENTATIONS flag requires `pipelineExecutableInfo` feature (params[%d])", i); + return {}; + } + retval += validation; } return retval; diff --git a/src/nbl/video/CVulkanComputePipeline.cpp b/src/nbl/video/CVulkanComputePipeline.cpp index 8335bae4fc..b5b18d677d 100644 --- a/src/nbl/video/CVulkanComputePipeline.cpp +++ b/src/nbl/video/CVulkanComputePipeline.cpp @@ -1,6 +1,7 @@ #include "nbl/video/CVulkanComputePipeline.h" #include "nbl/video/CVulkanLogicalDevice.h" +#include "nbl/video/CVulkanPipelineExecutableInfo.h" namespace nbl::video { @@ -12,6 +13,12 @@ CVulkanComputePipeline::~CVulkanComputePipeline() vk->vk.vkDestroyPipeline(vulkanDevice->getInternalObject(), m_pipeline, nullptr); } +void CVulkanComputePipeline::populateExecutableInfo(bool includeInternalRepresentations) +{ + const CVulkanLogicalDevice* vulkanDevice = static_cast(getOriginDevice()); + populateExecutableInfoFromVulkan(m_executableInfo, vulkanDevice->getFunctionTable(), vulkanDevice->getInternalObject(), m_pipeline, includeInternalRepresentations); +} + void CVulkanComputePipeline::setObjectDebugName(const char* label) const { IBackendObject::setObjectDebugName(label); diff --git a/src/nbl/video/CVulkanComputePipeline.h b/src/nbl/video/CVulkanComputePipeline.h index 89077f9a9a..e2a56d5450 100644 --- a/src/nbl/video/CVulkanComputePipeline.h +++ b/src/nbl/video/CVulkanComputePipeline.h @@ -22,7 +22,9 @@ class CVulkanComputePipeline final : public IGPUComputePipeline inline const void* getNativeHandle() const override { return &m_pipeline; } inline VkPipeline getInternalObject() const { return m_pipeline; } - + + void populateExecutableInfo(bool includeInternalRepresentations); + void setObjectDebugName(const char* label) const override; private: diff --git a/src/nbl/video/CVulkanGraphicsPipeline.cpp b/src/nbl/video/CVulkanGraphicsPipeline.cpp index a57dbfc83a..9872725bd4 100644 --- a/src/nbl/video/CVulkanGraphicsPipeline.cpp +++ b/src/nbl/video/CVulkanGraphicsPipeline.cpp @@ -1,10 +1,17 @@ #include "nbl/video/CVulkanGraphicsPipeline.h" #include "nbl/video/CVulkanLogicalDevice.h" +#include "CVulkanPipelineExecutableInfo.h" namespace nbl::video { +void CVulkanGraphicsPipeline::populateExecutableInfo(bool includeInternalRepresentations) +{ + const CVulkanLogicalDevice* vulkanDevice = static_cast(getOriginDevice()); + populateExecutableInfoFromVulkan(m_executableInfo, vulkanDevice->getFunctionTable(), vulkanDevice->getInternalObject(), m_vkPipeline, includeInternalRepresentations); +} + CVulkanGraphicsPipeline::~CVulkanGraphicsPipeline() { const CVulkanLogicalDevice* vulkanDevice = static_cast(getOriginDevice()); diff --git a/src/nbl/video/CVulkanGraphicsPipeline.h b/src/nbl/video/CVulkanGraphicsPipeline.h index 1b99e58dd6..4a11610b20 100644 --- a/src/nbl/video/CVulkanGraphicsPipeline.h +++ b/src/nbl/video/CVulkanGraphicsPipeline.h @@ -18,6 +18,8 @@ class CVulkanGraphicsPipeline final : public IGPUGraphicsPipeline inline VkPipeline getInternalObject() const {return m_vkPipeline;} + void populateExecutableInfo(bool includeInternalRepresentations); + private: ~CVulkanGraphicsPipeline(); diff --git a/src/nbl/video/CVulkanLogicalDevice.cpp b/src/nbl/video/CVulkanLogicalDevice.cpp index 46b79a7094..913bfc6318 100644 --- a/src/nbl/video/CVulkanLogicalDevice.cpp +++ b/src/nbl/video/CVulkanLogicalDevice.cpp @@ -1190,9 +1190,16 @@ void CVulkanLogicalDevice::createComputePipelines_impl( const VkPipeline vk_pipeline = vk_pipelines[i]; // break the lifetime cause of the aliasing std::uninitialized_default_construct_n(output+i,1); - output[i] = core::make_smart_refctd_ptr( + auto pipeline = core::make_smart_refctd_ptr( info,vk_pipeline ); + if (info.flags.hasFlags(IGPUComputePipeline::SCreationParams::FLAGS::CAPTURE_STATISTICS)) + { + pipeline->populateExecutableInfo(info.flags.hasFlags(IGPUComputePipeline::SCreationParams::FLAGS::CAPTURE_INTERNAL_REPRESENTATIONS)); + if (pipeline->getExecutableInfo().empty()) + m_logger.log("Driver returned 0 executables for pipeline created with CAPTURE_STATISTICS flag. This pipeline type may not be supported by the driver's VK_KHR_pipeline_executable_properties implementation.", system::ILogger::ELL_WARNING); + } + output[i] = std::move(pipeline); debugNameBuilder.str(""); const auto& specInfo = createInfos[i].shader; debugNameBuilder << specInfo.shader->getFilepathHint() << "(" << specInfo.entryPoint << "," << hlsl::ShaderStage::ESS_COMPUTE << ")\n"; @@ -1446,7 +1453,14 @@ void CVulkanLogicalDevice::createGraphicsPipelines_impl( const VkPipeline vk_pipeline = vk_pipelines[i]; // break the lifetime cause of the aliasing std::uninitialized_default_construct_n(output+i,1); - output[i] = core::make_smart_refctd_ptr(createInfos[i],vk_pipeline); + auto pipeline = core::make_smart_refctd_ptr(createInfos[i],vk_pipeline); + if (createInfo.flags.hasFlags(IGPUGraphicsPipeline::SCreationParams::FLAGS::CAPTURE_STATISTICS)) + { + pipeline->populateExecutableInfo(createInfo.flags.hasFlags(IGPUGraphicsPipeline::SCreationParams::FLAGS::CAPTURE_INTERNAL_REPRESENTATIONS)); + if (pipeline->getExecutableInfo().empty()) + m_logger.log("Driver returned 0 executables for pipeline created with CAPTURE_STATISTICS flag. This pipeline type may not be supported by the driver's VK_KHR_pipeline_executable_properties implementation.", system::ILogger::ELL_WARNING); + } + output[i] = std::move(pipeline); debugNameBuilder.str(""); auto buildDebugName = [&](const IGPUPipelineBase::SShaderSpecInfo& spec, hlsl::ShaderStage stage) { @@ -1651,11 +1665,19 @@ void CVulkanLogicalDevice::createRayTracingPipelines_impl( const auto success = m_devf.vk.vkGetRayTracingShaderGroupHandlesKHR(m_vkdev, vk_pipeline, 0, handleCount, dataSize, shaderGroupHandles->data()) == VK_SUCCESS; assert(success); - output[i] = core::make_smart_refctd_ptr( + auto pipeline = core::make_smart_refctd_ptr( createInfos[i], vk_pipeline, std::move(shaderGroupHandles) ); + + if (info.cached.flags.hasFlags(IGPURayTracingPipeline::SCreationParams::FLAGS::CAPTURE_STATISTICS)) + { + pipeline->populateExecutableInfo(info.cached.flags.hasFlags(IGPURayTracingPipeline::SCreationParams::FLAGS::CAPTURE_INTERNAL_REPRESENTATIONS)); + if (pipeline->getExecutableInfo().empty()) + m_logger.log("Driver returned 0 executables for pipeline created with CAPTURE_STATISTICS flag. This pipeline type may not be supported by the driver's VK_KHR_pipeline_executable_properties implementation.", system::ILogger::ELL_WARNING); + } + output[i] = std::move(pipeline); } } else diff --git a/src/nbl/video/CVulkanPipelineExecutableInfo.h b/src/nbl/video/CVulkanPipelineExecutableInfo.h new file mode 100644 index 0000000000..4cdf1f194f --- /dev/null +++ b/src/nbl/video/CVulkanPipelineExecutableInfo.h @@ -0,0 +1,149 @@ +#ifndef _NBL_VIDEO_C_VULKAN_PIPELINE_EXECUTABLE_INFO_H_INCLUDED_ +#define _NBL_VIDEO_C_VULKAN_PIPELINE_EXECUTABLE_INFO_H_INCLUDED_ + +#include "nbl/video/IGPUPipeline.h" +#include "nbl/video/CVulkanDeviceFunctionTable.h" + +#include + +namespace nbl::video +{ + +inline void populateExecutableInfoFromVulkan(core::vector& outInfo, const CVulkanDeviceFunctionTable* vk, VkDevice vkDevice, VkPipeline vkPipeline, bool includeInternalRepresentations) +{ + VkPipelineInfoKHR pipelineInfo = {VK_STRUCTURE_TYPE_PIPELINE_INFO_KHR, nullptr}; + pipelineInfo.pipeline = vkPipeline; + + // Enumerate executables + uint32_t executableCount = 0; + vk->vk.vkGetPipelineExecutablePropertiesKHR(vkDevice, &pipelineInfo, &executableCount, nullptr); + + if (executableCount == 0) + return; + + core::vector properties(executableCount); + for (uint32_t i = 0; i < executableCount; ++i) + properties[i] = {VK_STRUCTURE_TYPE_PIPELINE_EXECUTABLE_PROPERTIES_KHR, nullptr}; + vk->vk.vkGetPipelineExecutablePropertiesKHR(vkDevice, &pipelineInfo, &executableCount, properties.data()); + + outInfo.resize(executableCount); + + for (uint32_t i = 0; i < executableCount; ++i) + { + const auto& prop = properties[i]; + auto& info = outInfo[i]; + + info.name = prop.name; + info.description = prop.description; + info.stages = static_cast(prop.stages); + info.subgroupSize = prop.subgroupSize; + + VkPipelineExecutableInfoKHR execInfo = {VK_STRUCTURE_TYPE_PIPELINE_EXECUTABLE_INFO_KHR, nullptr}; + execInfo.pipeline = vkPipeline; + execInfo.executableIndex = i; + + uint32_t statCount = 0; + vk->vk.vkGetPipelineExecutableStatisticsKHR(vkDevice, &execInfo, &statCount, nullptr); + + if (statCount > 0) + { + core::vector stats(statCount); + for (uint32_t s = 0; s < statCount; ++s) + stats[s] = {VK_STRUCTURE_TYPE_PIPELINE_EXECUTABLE_STATISTIC_KHR, nullptr}; + vk->vk.vkGetPipelineExecutableStatisticsKHR(vkDevice, &execInfo, &statCount, stats.data()); + + // First pass: format name:value pairs and find max width for alignment + core::vector nameValues(statCount); + size_t maxNameValueLen = 0; + for (uint32_t s = 0; s < statCount; ++s) + { + const auto& stat = stats[s]; + std::string value; + switch (stat.format) + { + case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_BOOL32_KHR: + value = stat.value.b32 ? "true" : "false"; + break; + case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_INT64_KHR: + value = std::to_string(stat.value.i64); + break; + case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR: + value = std::to_string(stat.value.u64); + break; + case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_FLOAT64_KHR: + value = std::to_string(stat.value.f64); + break; + default: + value = ""; + break; + } + nameValues[s] = std::string(stat.name) + ": " + value; + maxNameValueLen = std::max(maxNameValueLen, nameValues[s].size()); + } + + // Second pass: emit with aligned columns + std::string& statsStr = info.statistics; + for (uint32_t s = 0; s < statCount; ++s) + { + statsStr += nameValues[s]; + statsStr.append(maxNameValueLen - nameValues[s].size() + 4, ' '); + statsStr += "// "; + statsStr += stats[s].description; + statsStr += "\n"; + } + } + + // Internal representations + if (includeInternalRepresentations) + { + uint32_t irCount = 0; + vk->vk.vkGetPipelineExecutableInternalRepresentationsKHR(vkDevice, &execInfo, &irCount, nullptr); + + if (irCount > 0) + { + core::vector irs(irCount); + for (uint32_t r = 0; r < irCount; ++r) + irs[r] = {VK_STRUCTURE_TYPE_PIPELINE_EXECUTABLE_INTERNAL_REPRESENTATION_KHR, nullptr}; + + // First call to get sizes + vk->vk.vkGetPipelineExecutableInternalRepresentationsKHR(vkDevice, &execInfo, &irCount, irs.data()); + + // Allocate data buffers and second call to get data + core::vector> irData(irCount); + for (uint32_t r = 0; r < irCount; ++r) + { + irData[r].resize(irs[r].dataSize); + irs[r].pData = irData[r].data(); + } + + vk->vk.vkGetPipelineExecutableInternalRepresentationsKHR(vkDevice, &execInfo, &irCount, irs.data()); + + std::string& irStr = info.internalRepresentations; + for (uint32_t r = 0; r < irCount; ++r) + { + irStr += "---- "; + irStr += irs[r].name; + irStr += " ----\n"; + irStr += irs[r].description; + irStr += "\n"; + if (irs[r].isText) + { + auto* str = static_cast(irs[r].pData); + irStr.append(str, irs[r].dataSize > 0 ? irs[r].dataSize - 1 : 0); + irStr += "\n"; + } + else + { + irStr += "[binary data, "; + irStr += std::to_string(irs[r].dataSize); + irStr += " bytes]\n"; + } + } + } + } + } +} + +} + +#endif diff --git a/src/nbl/video/CVulkanRayTracingPipeline.cpp b/src/nbl/video/CVulkanRayTracingPipeline.cpp index 960d78428a..48a6b4a92a 100644 --- a/src/nbl/video/CVulkanRayTracingPipeline.cpp +++ b/src/nbl/video/CVulkanRayTracingPipeline.cpp @@ -3,6 +3,7 @@ #include "nbl/video/CVulkanRayTracingPipeline.h" #include "nbl/video/CVulkanLogicalDevice.h" #include "nbl/video/IGPURayTracingPipeline.h" +#include "nbl/video/CVulkanPipelineExecutableInfo.h" #include @@ -78,6 +79,12 @@ namespace nbl::video vk->vk.vkDestroyPipeline(vulkanDevice->getInternalObject(), m_vkPipeline, nullptr); } + void CVulkanRayTracingPipeline::populateExecutableInfo(bool includeInternalRepresentations) + { + const CVulkanLogicalDevice* vulkanDevice = static_cast(getOriginDevice()); + populateExecutableInfoFromVulkan(m_executableInfo, vulkanDevice->getFunctionTable(), vulkanDevice->getInternalObject(), m_vkPipeline, includeInternalRepresentations); + } + const IGPURayTracingPipeline::SShaderGroupHandle& CVulkanRayTracingPipeline::getRaygen() const { return m_shaderGroupHandles->operator[](getRaygenIndex()); diff --git a/src/nbl/video/device_capabilities/device_features.json b/src/nbl/video/device_capabilities/device_features.json index 5e4775e9b4..f3989e26ee 100644 --- a/src/nbl/video/device_capabilities/device_features.json +++ b/src/nbl/video/device_capabilities/device_features.json @@ -2922,7 +2922,6 @@ "name": "pipelineExecutableInfo", "value": false, "comment": [ - "[TODO] need impl", "PipelineExecutablePropertiesFeaturesKHR", "VK_KHR_pipeline_executable_properties" ]