fix: store image implicit args in bindless slot 1

bindless slot 0: image
bindless slot 1: image implicit args
bindless slot 2: redescribed image

Redescribed image moved one slot higher.
Separate allocation is created for new bindless slot.

Related-To: NEO-9740
Signed-off-by: Fabian Zwolinski <fabian.zwolinski@intel.com>

Author: fzwolinski

level_zero/core/source/context/context_imp.cpp

@@ -513,18 +513,26 @@ ze_result_t ContextImp::evictMemory(ze_device_handle_t hDevice, void *ptr, size_
 
 ze_result_t ContextImp::makeImageResident(ze_device_handle_t hDevice, ze_image_handle_t hImage) {
     auto alloc = Image::fromHandle(hImage)->getAllocation();
+    auto implicitArgsAlloc = Image::fromHandle(hImage)->getImplicitArgsAllocation();
 
     NEO::Device *neoDevice = L0::Device::fromHandle(hDevice)->getNEODevice();
     NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get();
     auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&alloc, 1));
+    if (implicitArgsAlloc) {
+        memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&implicitArgsAlloc, 1));
+    }
     return changeMemoryOperationStatusToL0ResultType(success);
 }
 ze_result_t ContextImp::evictImage(ze_device_handle_t hDevice, ze_image_handle_t hImage) {
     auto alloc = Image::fromHandle(hImage)->getAllocation();
+    auto implicitArgsAlloc = Image::fromHandle(hImage)->getImplicitArgsAllocation();
 
     NEO::Device *neoDevice = L0::Device::fromHandle(hDevice)->getNEODevice();
     NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get();
     auto success = memoryOperationsIface->evict(neoDevice, *alloc);
+    if (implicitArgsAlloc) {
+        memoryOperationsIface->evict(neoDevice, *implicitArgsAlloc);
+    }
     return changeMemoryOperationStatusToL0ResultType(success);
 }
 

level_zero/core/source/image/image.h

@@ -36,10 +36,12 @@ struct Image : _ze_image_handle_t {
     virtual ze_result_t createView(Device *device, const ze_image_desc_t *desc, ze_image_handle_t *pImage) = 0;
 
     virtual NEO::GraphicsAllocation *getAllocation() = 0;
+    virtual NEO::GraphicsAllocation *getImplicitArgsAllocation() = 0;
     virtual void copySurfaceStateToSSH(void *surfaceStateHeap,
                                        const uint32_t surfaceStateOffset,
                                        bool isMediaBlockArg) = 0;
     virtual void copyRedescribedSurfaceStateToSSH(void *surfaceStateHeap, const uint32_t surfaceStateOffset) = 0;
+    virtual void copyImplicitArgsSurfaceStateToSSH(void *surfaceStateHeap, const uint32_t surfaceStateOffset) = 0;
     virtual NEO::ImageInfo getImageInfo() = 0;
     virtual ze_image_desc_t getImageDesc() = 0;
     virtual ze_result_t getMemoryProperties(ze_image_memory_properties_exp_t *pMemoryProperties) = 0;

level_zero/core/source/image/image_hw.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2022 Intel Corporation
+ * Copyright (C) 2020-2023 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -22,6 +22,7 @@ struct ImageCoreFamily : public ImageImp {
     ze_result_t initialize(Device *device, const ze_image_desc_t *desc) override;
     void copySurfaceStateToSSH(void *surfaceStateHeap, const uint32_t surfaceStateOffset, bool isMediaBlockArg) override;
     void copyRedescribedSurfaceStateToSSH(void *surfaceStateHeap, const uint32_t surfaceStateOffset) override;
+    void copyImplicitArgsSurfaceStateToSSH(void *surfaceStateHeap, const uint32_t surfaceStateOffset) override;
     bool isMediaFormat(const ze_image_format_layout_t layout) {
         if (layout == ze_image_format_layout_t::ZE_IMAGE_FORMAT_LAYOUT_NV12 ||
             layout == ze_image_format_layout_t::ZE_IMAGE_FORMAT_LAYOUT_P010 ||
@@ -49,6 +50,7 @@ struct ImageCoreFamily : public ImageImp {
     bool isSuitableForCompression(const StructuresLookupTable &structuresLookupTable, const NEO::ImageInfo &imgInfo);
 
     RENDER_SURFACE_STATE surfaceState;
+    RENDER_SURFACE_STATE implicitArgsSurfaceState;
     RENDER_SURFACE_STATE redescribedSurfaceState;
 };
 

level_zero/core/source/image/image_hw.inl

@@ -11,24 +11,33 @@
 #include "shared/source/device/device.h"
 #include "shared/source/execution_environment/root_device_environment.h"
 #include "shared/source/gmm_helper/gmm.h"
+#include "shared/source/helpers/api_specific_config.h"
 #include "shared/source/helpers/basic_math.h"
+#include "shared/source/helpers/gfx_core_helper.h"
 #include "shared/source/helpers/surface_format_info.h"
 #include "shared/source/image/image_surface_state.h"
 #include "shared/source/memory_manager/allocation_properties.h"
 #include "shared/source/memory_manager/memory_manager.h"
+#include "shared/source/release_helper/release_helper.h"
 
 #include "level_zero/core/source/device/device.h"
 #include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h"
 #include "level_zero/core/source/helpers/properties_parser.h"
+#include "level_zero/core/source/image/image_format_desc_helper.h"
 #include "level_zero/core/source/image/image_formats.h"
 #include "level_zero/core/source/image/image_hw.h"
 
+#include "encode_surface_state_args.h"
+
 namespace L0 {
 
 template <GFXCORE_FAMILY gfxCoreFamily>
 ze_result_t ImageCoreFamily<gfxCoreFamily>::initialize(Device *device, const ze_image_desc_t *desc) {
     using RENDER_SURFACE_STATE = typename GfxFamily::RENDER_SURFACE_STATE;
 
+    const auto &rootDeviceEnvironment = device->getNEODevice()->getRootDeviceEnvironment();
+    const bool isBindlessMode = rootDeviceEnvironment.getReleaseHelper() ? NEO::ApiSpecificConfig::getBindlessMode(rootDeviceEnvironment.getReleaseHelper()) : false;
+
     StructuresLookupTable lookupTable = {};
 
     lookupTable.areImageProperties = true;
@@ -104,8 +113,13 @@ ze_result_t ImageCoreFamily<gfxCoreFamily>::initialize(Device *device, const ze_
         }
     }
 
+    if (isBindlessMode) {
+        NEO::AllocationProperties imgImplicitArgsAllocProperties(device->getRootDeviceIndex(), NEO::ImageImplicitArgs::getSize(), NEO::AllocationType::buffer, device->getNEODevice()->getDeviceBitfield());
+        implicitArgsAllocation = device->getNEODevice()->getMemoryManager()->allocateGraphicsMemoryWithProperties(imgImplicitArgsAllocProperties);
+    }
+
     auto gmm = this->allocation->getDefaultGmm();
-    auto gmmHelper = static_cast<const NEO::RootDeviceEnvironment &>(device->getNEODevice()->getRootDeviceEnvironment()).getGmmHelper();
+    auto gmmHelper = static_cast<const NEO::RootDeviceEnvironment &>(rootDeviceEnvironment).getGmmHelper();
 
     if (gmm != nullptr) {
         NEO::ImagePlane yuvPlaneType = NEO::ImagePlane::noPlane;
@@ -157,6 +171,53 @@ ze_result_t ImageCoreFamily<gfxCoreFamily>::initialize(Device *device, const ze_
             NEO::EncodeSurfaceState<GfxFamily>::setImageAuxParamsForCCS(&surfaceState, gmm);
         }
     }
+
+    if (isBindlessMode && implicitArgsAllocation) {
+        implicitArgsSurfaceState = GfxFamily::cmdInitRenderSurfaceState;
+
+        auto clChannelType = getClChannelDataType(imageFormatDesc.format);
+        auto clChannelOrder = getClChannelOrder(imageFormatDesc.format);
+
+        NEO::ImageImplicitArgs imageImplicitArgs{};
+        imageImplicitArgs.structVersion = 0;
+
+        imageImplicitArgs.imageWidth = imgInfo.imgDesc.imageWidth;
+        imageImplicitArgs.imageHeight = imgInfo.imgDesc.imageHeight;
+        imageImplicitArgs.imageDepth = imgInfo.imgDesc.imageDepth;
+        imageImplicitArgs.imageArraySize = imgInfo.imgDesc.imageArraySize;
+        imageImplicitArgs.numSamples = imgInfo.imgDesc.numSamples;
+        imageImplicitArgs.channelType = clChannelType;
+        imageImplicitArgs.channelOrder = clChannelOrder;
+        imageImplicitArgs.numMipLevels = imgInfo.imgDesc.numMipLevels;
+
+        auto pixelSize = imgInfo.surfaceFormat->imageElementSizeInBytes;
+        imageImplicitArgs.flatBaseOffset = implicitArgsAllocation->getGpuAddress();
+        imageImplicitArgs.flatWidth = (imgInfo.imgDesc.imageWidth * pixelSize) - 1u;
+        imageImplicitArgs.flagHeight = (imgInfo.imgDesc.imageHeight * pixelSize) - 1u;
+        imageImplicitArgs.flatPitch = imgInfo.imgDesc.imageRowPitch - 1u;
+
+        const auto &productHelper = rootDeviceEnvironment.getHelper<NEO::ProductHelper>();
+        NEO::MemoryTransferHelper::transferMemoryToAllocation(productHelper.isBlitCopyRequiredForLocalMemory(rootDeviceEnvironment, *implicitArgsAllocation), *this->device->getNEODevice(), implicitArgsAllocation, 0u, &imageImplicitArgs, NEO::ImageImplicitArgs::getSize());
+
+        {
+            auto &gfxCoreHelper = this->device->getGfxCoreHelper();
+
+            NEO::EncodeSurfaceStateArgs args;
+            args.outMemory = &implicitArgsSurfaceState;
+            args.size = NEO::ImageImplicitArgs::getSize();
+            args.graphicsAddress = implicitArgsAllocation->getGpuAddress();
+            args.gmmHelper = gmmHelper;
+            args.allocation = implicitArgsAllocation;
+            args.numAvailableDevices = this->device->getNEODevice()->getNumGenericSubDevices();
+            args.areMultipleSubDevicesInContext = args.numAvailableDevices > 1;
+            args.mocs = gfxCoreHelper.getMocsIndex(*args.gmmHelper, true, false) << 1;
+            args.implicitScaling = this->device->isImplicitScalingCapable();
+            args.isDebuggerActive = this->device->getNEODevice()->getDebugger() != nullptr;
+
+            gfxCoreHelper.encodeBufferSurfaceState(args);
+        }
+    }
+
     {
         const uint32_t exponent = Math::log2(imgInfo.surfaceFormat->imageElementSizeInBytes);
         DEBUG_BREAK_IF(exponent >= 5u);
@@ -232,6 +293,18 @@ void ImageCoreFamily<gfxCoreFamily>::copyRedescribedSurfaceStateToSSH(void *surf
              &redescribedSurfaceState, sizeof(RENDER_SURFACE_STATE));
 }
 
+template <GFXCORE_FAMILY gfxCoreFamily>
+void ImageCoreFamily<gfxCoreFamily>::copyImplicitArgsSurfaceStateToSSH(void *surfaceStateHeap,
+                                                                       const uint32_t surfaceStateOffset) {
+    using GfxFamily = typename NEO::GfxFamilyMapper<gfxCoreFamily>::GfxFamily;
+    using RENDER_SURFACE_STATE = typename GfxFamily::RENDER_SURFACE_STATE;
+
+    // Copy the image's surface state into position in the provided surface state heap
+    auto destSurfaceState = ptrOffset(surfaceStateHeap, surfaceStateOffset);
+    memcpy_s(destSurfaceState, sizeof(RENDER_SURFACE_STATE),
+             &implicitArgsSurfaceState, sizeof(RENDER_SURFACE_STATE));
+}
+
 template <GFXCORE_FAMILY gfxCoreFamily>
 bool ImageCoreFamily<gfxCoreFamily>::isSuitableForCompression(const StructuresLookupTable &structuresLookupTable, const NEO::ImageInfo &imgInfo) {
     auto &hwInfo = device->getHwInfo();

level_zero/core/source/image/image_imp.cpp

@@ -31,8 +31,13 @@ ImageImp::~ImageImp() {
             this->device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[this->allocation->getRootDeviceIndex()]->getBindlessHeapsHelper()->releaseSSToReusePool(*bindlessInfo);
         }
     }
-    if (!isImageView() && this->device != nullptr) {
-        this->device->getNEODevice()->getMemoryManager()->freeGraphicsMemory(this->allocation);
+    if (this->device != nullptr) {
+        if (!isImageView()) {
+            this->device->getNEODevice()->getMemoryManager()->freeGraphicsMemory(this->allocation);
+        }
+        if (implicitArgsAllocation) {
+            this->device->getNEODevice()->getMemoryManager()->freeGraphicsMemory(this->implicitArgsAllocation);
+        }
     }
 }
 
@@ -100,7 +105,7 @@ ze_result_t ImageImp::allocateBindlessSlot() {
 
     if (bindlessHelper && !bindlessInfo) {
         auto &gfxCoreHelper = this->device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[allocation->getRootDeviceIndex()]->getHelper<NEO::GfxCoreHelper>();
-        const auto surfStateCount = 2;
+        const auto surfStateCount = 3;
         auto surfaceStateSize = surfStateCount * gfxCoreHelper.getRenderSurfaceStateSize();
 
         auto surfaceStateInfo = bindlessHelper->allocateSSInHeap(surfaceStateSize, allocation, NEO::BindlessHeapsHelper::globalSsh);

level_zero/core/source/image/image_imp.h

@@ -9,6 +9,7 @@
 
 #include "shared/source/helpers/non_copyable_or_moveable.h"
 #include "shared/source/helpers/surface_format_info.h"
+#include "shared/source/memory_manager/graphics_allocation.h"
 
 #include "level_zero/core/source/image/image.h"
 
@@ -26,6 +27,7 @@ struct ImageImp : public Image, NEO::NonCopyableOrMovableClass {
     ~ImageImp() override;
 
     NEO::GraphicsAllocation *getAllocation() override { return allocation; }
+    NEO::GraphicsAllocation *getImplicitArgsAllocation() override { return implicitArgsAllocation; }
     NEO::ImageInfo getImageInfo() override { return imgInfo; }
     ze_image_desc_t getImageDesc() override {
         return imageFormatDesc;
@@ -52,6 +54,7 @@ struct ImageImp : public Image, NEO::NonCopyableOrMovableClass {
     Device *device = nullptr;
     NEO::ImageInfo imgInfo = {};
     NEO::GraphicsAllocation *allocation = nullptr;
+    NEO::GraphicsAllocation *implicitArgsAllocation = nullptr;
     ze_image_desc_t imageFormatDesc = {};
     std::optional<ze_image_desc_t> sourceImageFormatDesc = {};
     std::unique_ptr<NEO::SurfaceStateInHeapInfo> bindlessInfo;

level_zero/core/source/kernel/kernel_imp.cpp

@@ -571,12 +571,12 @@ ze_result_t KernelImp::setArgRedescribedImage(uint32_t argIndex, ze_image_handle
 
             auto ssInHeap = image->getBindlessSlot();
             auto patchLocation = ptrOffset(getCrossThreadData(), arg.bindless);
-            // redescribed image's surface state is after image's state
-            auto bindlessSlotOffset = ssInHeap->surfaceStateOffset + surfaceStateSize;
+            // redescribed image's surface state is after image's implicit args
+            auto bindlessSlotOffset = ssInHeap->surfaceStateOffset + surfaceStateSize * 2;
             auto patchValue = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue(static_cast<uint32_t>(bindlessSlotOffset));
             patchWithRequiredSize(const_cast<uint8_t *>(patchLocation), sizeof(patchValue), patchValue);
 
-            image->copyRedescribedSurfaceStateToSSH(ptrOffset(ssInHeap->ssPtr, surfaceStateSize), 0u);
+            image->copyRedescribedSurfaceStateToSSH(ptrOffset(ssInHeap->ssPtr, surfaceStateSize * 2), 0u);
             isBindlessOffsetSet[argIndex] = true;
             this->residencyContainer.push_back(ssInHeap->heapAllocation);
         } else {
@@ -589,6 +589,10 @@ ze_result_t KernelImp::setArgRedescribedImage(uint32_t argIndex, ze_image_handle
     }
     residencyContainer[argIndex] = image->getAllocation();
 
+    if (image->getImplicitArgsAllocation()) {
+        this->residencyContainer.push_back(image->getImplicitArgsAllocation());
+    }
+
     return ZE_RESULT_SUCCESS;
 }
 
@@ -778,6 +782,8 @@ ze_result_t KernelImp::setArgImage(uint32_t argIndex, size_t argSize, const void
             patchWithRequiredSize(const_cast<uint8_t *>(patchLocation), sizeof(patchValue), patchValue);
 
             image->copySurfaceStateToSSH(ssInHeap->ssPtr, 0u, isMediaBlockImage);
+            image->copyImplicitArgsSurfaceStateToSSH(ptrOffset(ssInHeap->ssPtr, surfaceStateSize), 0u);
+
             isBindlessOffsetSet[argIndex] = true;
             this->residencyContainer.push_back(ssInHeap->heapAllocation);
         } else {
@@ -791,6 +797,10 @@ ze_result_t KernelImp::setArgImage(uint32_t argIndex, size_t argSize, const void
 
     residencyContainer[argIndex] = image->getAllocation();
 
+    if (image->getImplicitArgsAllocation()) {
+        this->residencyContainer.push_back(image->getImplicitArgsAllocation());
+    }
+
     auto imageInfo = image->getImageInfo();
     auto clChannelType = getClChannelDataType(image->getImageDesc().format);
     auto clChannelOrder = getClChannelOrder(image->getImageDesc().format);

level_zero/core/test/unit_tests/mocks/mock_image.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2022 Intel Corporation
+ * Copyright (C) 2022-2023 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -25,6 +25,7 @@ struct WhiteBox<::L0::ImageCoreFamily<gfxCoreFamily>>
 
     ~WhiteBox() override {}
 
+    using BaseClass::implicitArgsSurfaceState;
     using BaseClass::redescribedSurfaceState;
     using BaseClass::surfaceState;
 };