PipelineStateBase.hpp

Go to the documentation of this file.

/*
 *  Copyright 2019-2021 Diligent Graphics LLC
 *  Copyright 2015-2019 Egor Yusov
 *  
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *  
 *      http://www.apache.org/licenses/LICENSE-2.0
 *  
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *  In no event and under no legal theory, whether in tort (including negligence), 
 *  contract, or otherwise, unless required by applicable law (such as deliberate 
 *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
 *  liable for any damages, including any direct, indirect, special, incidental, 
 *  or consequential damages of any character arising as a result of this License or 
 *  out of the use or inability to use the software (including but not limited to damages 
 *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and 
 *  all other commercial damages or losses), even if such Contributor has been advised 
 *  of the possibility of such damages.
 */
 
#pragma once
 
 
#include <array>
#include <unordered_map>
#include <unordered_set>
#include <cstring>
 
#include "PrivateConstants.h"
#include "PipelineState.h"
#include "DeviceObjectBase.hpp"
#include "STDAllocator.hpp"
#include "EngineMemory.h"
#include "GraphicsAccessories.hpp"
#include "FixedLinearAllocator.hpp"
#include "HashUtils.hpp"
#include "PipelineResourceSignatureBase.hpp"
 
namespace Diligent
{
 
// Validates graphics pipeline create attributes and throws an exception in case of an error.
void ValidateGraphicsPipelineCreateInfo(const GraphicsPipelineStateCreateInfo& CreateInfo,
                                        const DeviceFeatures&                  Features) noexcept(false);
 
// Validates compute pipeline create attributes and throws an exception in case of an error.
void ValidateComputePipelineCreateInfo(const ComputePipelineStateCreateInfo& CreateInfo,
                                       const DeviceFeatures&                 Features) noexcept(false);
 
// Validates ray-tracing pipeline create attributes and throws an exception in case of an error.
void ValidateRayTracingPipelineCreateInfo(IRenderDevice*                           pDevice,
                                          Uint32                                   MaxRecursion,
                                          const RayTracingPipelineStateCreateInfo& CreateInfo,
                                          const DeviceFeatures&                    Features) noexcept(false);
 
void ValidatePipelineResourceCompatibility(const PipelineResourceDesc& ResDesc,
                                           SHADER_RESOURCE_TYPE        Type,
                                           PIPELINE_RESOURCE_FLAGS     ResourceFlags,
                                           Uint32                      ArraySize,
                                           const char*                 ShaderName,
                                           const char*                 SignatureName) noexcept(false);
 
 
void CopyRTShaderGroupNames(std::unordered_map<HashMapStringKey, Uint32, HashMapStringKey::Hasher>& NameToGroupIndex,
                            const RayTracingPipelineStateCreateInfo&                                CreateInfo,
                            FixedLinearAllocator&                                                   MemPool) noexcept;
 
void CorrectGraphicsPipelineDesc(GraphicsPipelineDesc& GraphicsPipeline) noexcept;
 
 
static constexpr Uint32 InvalidPipelineResourceLayoutVariableIndex = ~0u;
Uint32 FindPipelineResourceLayoutVariable(const PipelineResourceLayoutDesc& LayoutDesc,
                                          const char*                       Name,
                                          SHADER_TYPE                       ShaderStage,
                                          const char*                       CombinedSamplerSuffix);
 
 
template <typename EngineImplTraits>
class PipelineStateBase : public DeviceObjectBase<typename EngineImplTraits::PipelineStateInterface, typename EngineImplTraits::RenderDeviceImplType, PipelineStateDesc>
{
private:
    // Base interface this class inherits (IPipelineStateD3D12, IPipelineStateVk, etc.)
    using BaseInterface = typename EngineImplTraits::PipelineStateInterface;
 
    // Render device implementation type (RenderDeviceD3D12Impl, RenderDeviceVkImpl, etc.).
    using RenderDeviceImplType = typename EngineImplTraits::RenderDeviceImplType;
 
    // Pipeline state implementation type (PipelineStateD3D12Impl, PipelineStateVkImpl, etc.).
    using PipelineStateImplType = typename EngineImplTraits::PipelineStateImplType;
 
    // Pipeline resource signature implementation type (PipelineResourceSignatureD3D12Impl, PipelineResourceSignatureVkImpl, etc.).
    using PipelineResourceSignatureImplType = typename EngineImplTraits::PipelineResourceSignatureImplType;
 
    using TDeviceObjectBase = DeviceObjectBase<BaseInterface, RenderDeviceImplType, PipelineStateDesc>;
 
    PipelineStateBase(IReferenceCounters*            pRefCounters,
                      RenderDeviceImplType*          pDevice,
                      const PipelineStateCreateInfo& CreateInfo,
                      bool                           bIsDeviceInternal = false) :
        TDeviceObjectBase{pRefCounters, pDevice, CreateInfo.PSODesc, bIsDeviceInternal},
        m_UsingImplicitSignature{CreateInfo.ppResourceSignatures == nullptr || CreateInfo.ResourceSignaturesCount == 0}
    {
        Uint64 DeviceQueuesMask = pDevice->GetCommandQueueMask();
        DEV_CHECK_ERR((this->m_Desc.CommandQueueMask & DeviceQueuesMask) != 0,
                      "No bits in the command queue mask (0x", std::hex, this->m_Desc.CommandQueueMask,
                      ") correspond to one of ", pDevice->GetCommandQueueCount(), " available device command queues.");
        this->m_Desc.CommandQueueMask &= DeviceQueuesMask;
    }
 
public:
 
    PipelineStateBase(IReferenceCounters*                    pRefCounters,
                      RenderDeviceImplType*                  pDevice,
                      const GraphicsPipelineStateCreateInfo& GraphicsPipelineCI,
                      bool                                   bIsDeviceInternal = false) :
        PipelineStateBase{pRefCounters, pDevice, static_cast<const PipelineStateCreateInfo&>(GraphicsPipelineCI), bIsDeviceInternal}
    {
        try
        {
            ValidateGraphicsPipelineCreateInfo(GraphicsPipelineCI, pDevice->GetDeviceCaps().Features);
        }
        catch (...)
        {
            Destruct();
            throw;
        }
    }
 
 
    PipelineStateBase(IReferenceCounters*                   pRefCounters,
                      RenderDeviceImplType*                 pDevice,
                      const ComputePipelineStateCreateInfo& ComputePipelineCI,
                      bool                                  bIsDeviceInternal = false) :
        PipelineStateBase{pRefCounters, pDevice, static_cast<const PipelineStateCreateInfo&>(ComputePipelineCI), bIsDeviceInternal}
    {
        try
        {
            ValidateComputePipelineCreateInfo(ComputePipelineCI, pDevice->GetDeviceCaps().Features);
        }
        catch (...)
        {
            Destruct();
            throw;
        }
    }
 
 
    PipelineStateBase(IReferenceCounters*                      pRefCounters,
                      RenderDeviceImplType*                    pDevice,
                      const RayTracingPipelineStateCreateInfo& RayTracingPipelineCI,
                      bool                                     bIsDeviceInternal = false) :
        PipelineStateBase{pRefCounters, pDevice, static_cast<const PipelineStateCreateInfo&>(RayTracingPipelineCI), bIsDeviceInternal}
    {
        try
        {
            ValidateRayTracingPipelineCreateInfo(pDevice, pDevice->GetProperties().MaxRayTracingRecursionDepth,
                                                 RayTracingPipelineCI, pDevice->GetDeviceCaps().Features);
        }
        catch (...)
        {
            Destruct();
            throw;
        }
    }
 
 
    ~PipelineStateBase()
    {
        /*
        auto &PipelineStateRegistry = static_cast<TRenderDeviceBase*>(this->GetDevice())->GetBSRegistry();
        auto &RasterizerStateRegistry = static_cast<TRenderDeviceBase*>(this->GetDevice())->GetRSRegistry();
        auto &DSSRegistry = static_cast<TRenderDeviceBase*>(this->GetDevice())->GetDSSRegistry();
        // StateObjectsRegistry::ReportDeletedObject() does not lock the registry, but only 
        // atomically increments the outstanding deleted objects counter.
        PipelineStateRegistry.ReportDeletedObject();
        RasterizerStateRegistry.ReportDeletedObject();
        DSSRegistry.ReportDeletedObject();
        */
        VERIFY(m_IsDestructed, "This object must be explicitly destructed with Destruct()");
    }
 
    void Destruct()
    {
        VERIFY(!m_IsDestructed, "This object has already been destructed");
 
        if (this->m_Desc.IsAnyGraphicsPipeline() && m_pGraphicsPipelineData != nullptr)
        {
            m_pGraphicsPipelineData->~GraphicsPipelineData();
        }
        else if (this->m_Desc.IsRayTracingPipeline() && m_pRayTracingPipelineData != nullptr)
        {
            m_pRayTracingPipelineData->~RayTracingPipelineData();
        }
 
        if (m_Signatures != nullptr)
        {
            for (Uint32 i = 0; i < m_SignatureCount; ++i)
                m_Signatures[i].~SignatureAutoPtrType();
            m_Signatures = nullptr;
        }
 
        if (m_pPipelineDataRawMem)
        {
            GetRawAllocator().Free(m_pPipelineDataRawMem);
            m_pPipelineDataRawMem = nullptr;
        }
#if DILIGENT_DEBUG
        m_IsDestructed = true;
#endif
    }
 
    IMPLEMENT_QUERY_INTERFACE_IN_PLACE(IID_PipelineState, TDeviceObjectBase)
 
    Uint32 GetBufferStride(Uint32 BufferSlot) const
    {
        VERIFY_EXPR(this->m_Desc.IsAnyGraphicsPipeline());
        return BufferSlot < m_pGraphicsPipelineData->BufferSlotsUsed ? m_pGraphicsPipelineData->pStrides[BufferSlot] : 0;
    }
 
    Uint32 GetNumBufferSlotsUsed() const
    {
        VERIFY_EXPR(this->m_Desc.IsAnyGraphicsPipeline());
        return m_pGraphicsPipelineData->BufferSlotsUsed;
    }
 
    RefCntAutoPtr<IRenderPass> const& GetRenderPassPtr() const
    {
        VERIFY_EXPR(this->m_Desc.IsAnyGraphicsPipeline());
        return m_pGraphicsPipelineData->pRenderPass;
    }
 
    RefCntAutoPtr<IRenderPass>& GetRenderPassPtr()
    {
        VERIFY_EXPR(this->m_Desc.IsAnyGraphicsPipeline());
        return m_pGraphicsPipelineData->pRenderPass;
    }
 
    virtual const GraphicsPipelineDesc& DILIGENT_CALL_TYPE GetGraphicsPipelineDesc() const override final
    {
        VERIFY_EXPR(this->m_Desc.IsAnyGraphicsPipeline());
        VERIFY_EXPR(m_pGraphicsPipelineData != nullptr);
        return m_pGraphicsPipelineData->Desc;
    }
 
    virtual const RayTracingPipelineDesc& DILIGENT_CALL_TYPE GetRayTracingPipelineDesc() const override final
    {
        VERIFY_EXPR(this->m_Desc.IsRayTracingPipeline());
        VERIFY_EXPR(m_pRayTracingPipelineData != nullptr);
        return m_pRayTracingPipelineData->Desc;
    }
 
    inline void CopyShaderHandle(const char* Name, void* pData, size_t DataSize) const
    {
        VERIFY_EXPR(this->m_Desc.IsRayTracingPipeline());
        VERIFY_EXPR(m_pRayTracingPipelineData != nullptr);
 
        const auto ShaderHandleSize = m_pRayTracingPipelineData->ShaderHandleSize;
        VERIFY(ShaderHandleSize <= DataSize, "DataSize (", DataSize, ") must be at least as large as the shader handle size (", ShaderHandleSize, ").");
 
        if (Name == nullptr || Name[0] == '\0')
        {
            // set shader binding to zero to skip shader execution
            std::memset(pData, 0, ShaderHandleSize);
            return;
        }
 
        auto iter = m_pRayTracingPipelineData->NameToGroupIndex.find(Name);
        if (iter != m_pRayTracingPipelineData->NameToGroupIndex.end())
        {
            VERIFY_EXPR(ShaderHandleSize * (iter->second + 1) <= m_pRayTracingPipelineData->ShaderDataSize);
            std::memcpy(pData, &m_pRayTracingPipelineData->ShaderHandles[ShaderHandleSize * iter->second], ShaderHandleSize);
            return;
        }
        UNEXPECTED("Can't find shader group '", Name, "'.");
    }
 
    virtual void DILIGENT_CALL_TYPE CreateShaderResourceBinding(IShaderResourceBinding** ppShaderResourceBinding,
                                                                bool                     InitStaticResources) override final
    {
        *ppShaderResourceBinding = nullptr;
 
        if (!m_UsingImplicitSignature)
        {
            LOG_ERROR_MESSAGE("IPipelineState::CreateShaderResourceBinding is not allowed for pipelines that use explicit "
                              "resource signatures. Use IPipelineResourceSignature::CreateShaderResourceBinding instead.");
            return;
        }
 
        return this->GetResourceSignature(0)->CreateShaderResourceBinding(ppShaderResourceBinding, InitStaticResources);
    }
 
    virtual IShaderResourceVariable* DILIGENT_CALL_TYPE GetStaticVariableByName(SHADER_TYPE ShaderType,
                                                                                const Char* Name) override final
    {
        if (!m_UsingImplicitSignature)
        {
            LOG_ERROR_MESSAGE("IPipelineState::CreateShaderResourceBinding is not allowed for pipelines that use explicit "
                              "resource signatures. Use IPipelineResourceSignature::GetStaticVariableByName instead.");
            return nullptr;
        }
 
        if ((m_ActiveShaderStages & ShaderType) == 0)
        {
            LOG_WARNING_MESSAGE("Unable to find static variable '", Name, "' in shader stage ", GetShaderTypeLiteralName(ShaderType),
                                " as the stage is inactive in PSO '", this->m_Desc.Name, "'.");
            return nullptr;
        }
 
        return this->GetResourceSignature(0)->GetStaticVariableByName(ShaderType, Name);
    }
 
    virtual IShaderResourceVariable* DILIGENT_CALL_TYPE GetStaticVariableByIndex(SHADER_TYPE ShaderType,
                                                                                 Uint32      Index) override final
    {
        if (!m_UsingImplicitSignature)
        {
            LOG_ERROR_MESSAGE("IPipelineState::GetStaticVariableByIndex is not allowed for pipelines that use explicit "
                              "resource signatures. Use IPipelineResourceSignature::GetStaticVariableByIndex instead.");
            return nullptr;
        }
 
        if ((m_ActiveShaderStages & ShaderType) == 0)
        {
            LOG_WARNING_MESSAGE("Unable to get static variable at index ", Index, " in shader stage ", GetShaderTypeLiteralName(ShaderType),
                                " as the stage is inactive in PSO '", this->m_Desc.Name, "'.");
            return nullptr;
        }
 
        return this->GetResourceSignature(0)->GetStaticVariableByIndex(ShaderType, Index);
    }
 
    virtual Uint32 DILIGENT_CALL_TYPE GetStaticVariableCount(SHADER_TYPE ShaderType) const override final
    {
        if (!m_UsingImplicitSignature)
        {
            LOG_ERROR_MESSAGE("IPipelineState::GetStaticVariableCount is not allowed for pipelines that use explicit "
                              "resource signatures. Use IPipelineResourceSignature::GetStaticVariableCount instead.");
            return 0;
        }
 
        if ((m_ActiveShaderStages & ShaderType) == 0)
        {
            LOG_WARNING_MESSAGE("Unable to get the number of static variables in shader stage ", GetShaderTypeLiteralName(ShaderType),
                                " as the stage is inactive in PSO '", this->m_Desc.Name, "'.");
            return 0;
        }
 
        return this->GetResourceSignature(0)->GetStaticVariableCount(ShaderType);
    }
 
    virtual void DILIGENT_CALL_TYPE BindStaticResources(Uint32 ShaderFlags, IResourceMapping* pResourceMapping, Uint32 Flags) override final
    {
        if (!m_UsingImplicitSignature)
        {
            LOG_ERROR_MESSAGE("IPipelineState::BindStaticResources is not allowed for pipelines that use explicit "
                              "resource signatures. Use IPipelineResourceSignature::BindStaticResources instead.");
            return;
        }
 
        return this->GetResourceSignature(0)->BindStaticResources(ShaderFlags, pResourceMapping, Flags);
    }
 
    virtual void DILIGENT_CALL_TYPE InitializeStaticSRBResources(IShaderResourceBinding* pSRB) const override final
    {
        if (!m_UsingImplicitSignature)
        {
            LOG_ERROR_MESSAGE("IPipelineState::InitializeStaticSRBResources is not allowed for pipelines that use explicit "
                              "resource signatures. Use IPipelineResourceSignature::InitializeStaticSRBResources instead.");
            return;
        }
 
        return this->GetResourceSignature(0)->InitializeStaticSRBResources(pSRB);
    }
 
    virtual Uint32 DILIGENT_CALL_TYPE GetResourceSignatureCount() const override final
    {
        return m_SignatureCount;
    }
 
    virtual PipelineResourceSignatureImplType* DILIGENT_CALL_TYPE GetResourceSignature(Uint32 Index) const override final
    {
        VERIFY_EXPR(Index < m_SignatureCount);
        return m_Signatures[Index];
    }
 
    virtual bool DILIGENT_CALL_TYPE IsCompatibleWith(const IPipelineState* pPSO) const override // May be overriden
    {
        DEV_CHECK_ERR(pPSO != nullptr, "pPSO must not be null");
 
        if (pPSO == this)
            return true;
 
        const auto& lhs = *static_cast<const PipelineStateImplType*>(this);
        const auto& rhs = *ValidatedCast<const PipelineStateImplType>(pPSO);
 
        const auto SignCount = lhs.GetResourceSignatureCount();
        if (SignCount != rhs.GetResourceSignatureCount())
            return false;
 
        for (Uint32 s = 0; s < SignCount; ++s)
        {
            const auto* pLhsSign = GetResourceSignature(s);
            const auto* pRhsSign = rhs.GetResourceSignature(s);
            if (!PipelineResourceSignatureImplType::SignaturesCompatible(pLhsSign, pRhsSign))
                return false;
        }
 
        return true;
    }
 
    SHADER_TYPE GetActiveShaderStages() const
    {
        return m_ActiveShaderStages;
    }
 
protected:
    using TNameToGroupIndexMap = std::unordered_map<HashMapStringKey, Uint32, HashMapStringKey::Hasher>;
 
    void ReserveSpaceForPipelineDesc(const GraphicsPipelineStateCreateInfo& CreateInfo,
                                     FixedLinearAllocator&                  MemPool) noexcept
    {
        MemPool.AddSpace<GraphicsPipelineData>();
        ReserveResourceLayout(CreateInfo.PSODesc.ResourceLayout, MemPool);
        ReserveResourceSignatures(CreateInfo, MemPool);
 
        const auto& InputLayout     = CreateInfo.GraphicsPipeline.InputLayout;
        Uint32      BufferSlotsUsed = 0;
        MemPool.AddSpace<LayoutElement>(InputLayout.NumElements);
        for (Uint32 i = 0; i < InputLayout.NumElements; ++i)
        {
            auto& LayoutElem = InputLayout.LayoutElements[i];
            MemPool.AddSpaceForString(LayoutElem.HLSLSemantic);
            BufferSlotsUsed = std::max(BufferSlotsUsed, LayoutElem.BufferSlot + 1);
        }
 
        MemPool.AddSpace<Uint32>(BufferSlotsUsed);
 
        static_assert(std::is_trivially_destructible<decltype(*InputLayout.LayoutElements)>::value, "Add destructor for this object to Destruct()");
    }
 
    void ReserveSpaceForPipelineDesc(const ComputePipelineStateCreateInfo& CreateInfo,
                                     FixedLinearAllocator&                 MemPool) noexcept
    {
        ReserveResourceLayout(CreateInfo.PSODesc.ResourceLayout, MemPool);
        ReserveResourceSignatures(CreateInfo, MemPool);
    }
 
    void ReserveSpaceForPipelineDesc(const RayTracingPipelineStateCreateInfo& CreateInfo,
                                     FixedLinearAllocator&                    MemPool) noexcept
    {
        size_t RTDataSize = sizeof(RayTracingPipelineData);
        // Reserve space for shader handles
        const auto ShaderHandleSize = this->m_pDevice->GetProperties().ShaderGroupHandleSize;
        RTDataSize += ShaderHandleSize * (CreateInfo.GeneralShaderCount + CreateInfo.TriangleHitShaderCount + CreateInfo.ProceduralHitShaderCount);
        // Extra bytes were reserved to avoid compiler errors on zero-sized arrays
        RTDataSize -= sizeof(RayTracingPipelineData::ShaderHandles);
        MemPool.AddSpace(RTDataSize, alignof(RayTracingPipelineData));
 
        for (Uint32 i = 0; i < CreateInfo.GeneralShaderCount; ++i)
        {
            MemPool.AddSpaceForString(CreateInfo.pGeneralShaders[i].Name);
        }
        for (Uint32 i = 0; i < CreateInfo.TriangleHitShaderCount; ++i)
        {
            MemPool.AddSpaceForString(CreateInfo.pTriangleHitShaders[i].Name);
        }
        for (Uint32 i = 0; i < CreateInfo.ProceduralHitShaderCount; ++i)
        {
            MemPool.AddSpaceForString(CreateInfo.pProceduralHitShaders[i].Name);
        }
 
        ReserveResourceLayout(CreateInfo.PSODesc.ResourceLayout, MemPool);
        ReserveResourceSignatures(CreateInfo, MemPool);
    }
 
 
    template <typename ShaderImplType, typename TShaderStages>
    void ExtractShaders(const GraphicsPipelineStateCreateInfo& CreateInfo,
                        TShaderStages&                         ShaderStages)
    {
        VERIFY_EXPR(this->m_Desc.IsAnyGraphicsPipeline());
 
        ShaderStages.clear();
        auto AddShaderStage = [&](IShader* pShader) {
            if (pShader != nullptr)
            {
                ShaderStages.emplace_back(ValidatedCast<ShaderImplType>(pShader));
                const auto ShaderType = pShader->GetDesc().ShaderType;
                VERIFY((m_ActiveShaderStages & ShaderType) == 0,
                       "Shader stage ", GetShaderTypeLiteralName(ShaderType), " has already been initialized in PSO '", this->m_Desc.Name, "'.");
                m_ActiveShaderStages |= ShaderType;
#ifdef DILIGENT_DEBUG
                for (Uint32 i = 0; i + 1 < ShaderStages.size(); ++i)
                    VERIFY_EXPR(GetShaderStageType(ShaderStages[i]) != ShaderType);
#endif
            }
        };
 
        switch (CreateInfo.PSODesc.PipelineType)
        {
            case PIPELINE_TYPE_GRAPHICS:
            {
                AddShaderStage(CreateInfo.pVS);
                AddShaderStage(CreateInfo.pHS);
                AddShaderStage(CreateInfo.pDS);
                AddShaderStage(CreateInfo.pGS);
                AddShaderStage(CreateInfo.pPS);
                VERIFY(CreateInfo.pVS != nullptr, "Vertex shader must not be null");
                break;
            }
 
            case PIPELINE_TYPE_MESH:
            {
                AddShaderStage(CreateInfo.pAS);
                AddShaderStage(CreateInfo.pMS);
                AddShaderStage(CreateInfo.pPS);
                VERIFY(CreateInfo.pMS != nullptr, "Mesh shader must not be null");
                break;
            }
 
            default:
                UNEXPECTED("unknown pipeline type");
        }
 
        VERIFY_EXPR(!ShaderStages.empty());
    }
 
    template <typename ShaderImplType, typename TShaderStages>
    void ExtractShaders(const ComputePipelineStateCreateInfo& CreateInfo,
                        TShaderStages&                        ShaderStages)
    {
        VERIFY_EXPR(this->m_Desc.IsComputePipeline());
 
        ShaderStages.clear();
 
        VERIFY_EXPR(CreateInfo.PSODesc.PipelineType == PIPELINE_TYPE_COMPUTE);
        VERIFY_EXPR(CreateInfo.pCS != nullptr);
        VERIFY_EXPR(CreateInfo.pCS->GetDesc().ShaderType == SHADER_TYPE_COMPUTE);
 
        ShaderStages.emplace_back(ValidatedCast<ShaderImplType>(CreateInfo.pCS));
        m_ActiveShaderStages = SHADER_TYPE_COMPUTE;
 
        VERIFY_EXPR(!ShaderStages.empty());
    }
 
    template <typename ShaderImplType, typename TShaderStages>
    void ExtractShaders(const RayTracingPipelineStateCreateInfo& CreateInfo,
                        TShaderStages&                           ShaderStages)
    {
        VERIFY_EXPR(this->m_Desc.IsRayTracingPipeline());
 
        std::unordered_set<IShader*> UniqueShaders;
 
        auto AddShader = [&ShaderStages, &UniqueShaders, this](IShader* pShader) {
            if (pShader != nullptr && UniqueShaders.insert(pShader).second)
            {
                const auto ShaderType = pShader->GetDesc().ShaderType;
                const auto StageInd   = GetShaderTypePipelineIndex(ShaderType, PIPELINE_TYPE_RAY_TRACING);
                auto&      Stage      = ShaderStages[StageInd];
                m_ActiveShaderStages |= ShaderType;
                Stage.Append(ValidatedCast<ShaderImplType>(pShader));
            }
        };
 
        ShaderStages.clear();
        ShaderStages.resize(MAX_SHADERS_IN_PIPELINE);
 
        for (Uint32 i = 0; i < CreateInfo.GeneralShaderCount; ++i)
        {
            AddShader(CreateInfo.pGeneralShaders[i].pShader);
        }
        for (Uint32 i = 0; i < CreateInfo.TriangleHitShaderCount; ++i)
        {
            AddShader(CreateInfo.pTriangleHitShaders[i].pClosestHitShader);
            AddShader(CreateInfo.pTriangleHitShaders[i].pAnyHitShader);
        }
        for (Uint32 i = 0; i < CreateInfo.ProceduralHitShaderCount; ++i)
        {
            AddShader(CreateInfo.pProceduralHitShaders[i].pIntersectionShader);
            AddShader(CreateInfo.pProceduralHitShaders[i].pClosestHitShader);
            AddShader(CreateInfo.pProceduralHitShaders[i].pAnyHitShader);
        }
 
        if (ShaderStages[GetShaderTypePipelineIndex(SHADER_TYPE_RAY_GEN, PIPELINE_TYPE_RAY_TRACING)].Count() == 0)
            LOG_ERROR_AND_THROW("At least one shader with type SHADER_TYPE_RAY_GEN must be provided.");
 
        // Remove empty stages
        for (auto iter = ShaderStages.begin(); iter != ShaderStages.end();)
        {
            if (iter->Count() == 0)
            {
                iter = ShaderStages.erase(iter);
                continue;
            }
 
            ++iter;
        }
 
        VERIFY_EXPR(!ShaderStages.empty());
    }
 
 
    void InitializePipelineDesc(const GraphicsPipelineStateCreateInfo& CreateInfo,
                                FixedLinearAllocator&                  MemPool)
    {
        this->m_pGraphicsPipelineData = MemPool.Construct<GraphicsPipelineData>();
        void* Ptr                     = MemPool.ReleaseOwnership();
        VERIFY_EXPR(Ptr == m_pPipelineDataRawMem);
 
        auto& GraphicsPipeline = this->m_pGraphicsPipelineData->Desc;
        auto& pRenderPass      = this->m_pGraphicsPipelineData->pRenderPass;
        auto& BufferSlotsUsed  = this->m_pGraphicsPipelineData->BufferSlotsUsed;
        auto& pStrides         = this->m_pGraphicsPipelineData->pStrides;
 
        GraphicsPipeline = CreateInfo.GraphicsPipeline;
        CorrectGraphicsPipelineDesc(GraphicsPipeline);
 
        CopyResourceLayout(CreateInfo.PSODesc.ResourceLayout, this->m_Desc.ResourceLayout, MemPool);
        CopyResourceSignatures(CreateInfo, MemPool);
 
        pRenderPass = GraphicsPipeline.pRenderPass;
        if (pRenderPass)
        {
            const auto& RPDesc = pRenderPass->GetDesc();
            VERIFY_EXPR(GraphicsPipeline.SubpassIndex < RPDesc.SubpassCount);
            const auto& Subpass = RPDesc.pSubpasses[GraphicsPipeline.SubpassIndex];
 
            GraphicsPipeline.NumRenderTargets = static_cast<Uint8>(Subpass.RenderTargetAttachmentCount);
            for (Uint32 rt = 0; rt < Subpass.RenderTargetAttachmentCount; ++rt)
            {
                const auto& RTAttachmentRef = Subpass.pRenderTargetAttachments[rt];
                if (RTAttachmentRef.AttachmentIndex != ATTACHMENT_UNUSED)
                {
                    VERIFY_EXPR(RTAttachmentRef.AttachmentIndex < RPDesc.AttachmentCount);
                    GraphicsPipeline.RTVFormats[rt] = RPDesc.pAttachments[RTAttachmentRef.AttachmentIndex].Format;
                }
            }
 
            if (Subpass.pDepthStencilAttachment != nullptr)
            {
                const auto& DSAttachmentRef = *Subpass.pDepthStencilAttachment;
                if (DSAttachmentRef.AttachmentIndex != ATTACHMENT_UNUSED)
                {
                    VERIFY_EXPR(DSAttachmentRef.AttachmentIndex < RPDesc.AttachmentCount);
                    GraphicsPipeline.DSVFormat = RPDesc.pAttachments[DSAttachmentRef.AttachmentIndex].Format;
                }
            }
        }
 
        const auto&    InputLayout     = GraphicsPipeline.InputLayout;
        LayoutElement* pLayoutElements = MemPool.ConstructArray<LayoutElement>(InputLayout.NumElements);
        for (size_t Elem = 0; Elem < InputLayout.NumElements; ++Elem)
        {
            const auto& SrcElem   = InputLayout.LayoutElements[Elem];
            pLayoutElements[Elem] = SrcElem;
            VERIFY_EXPR(SrcElem.HLSLSemantic != nullptr);
            pLayoutElements[Elem].HLSLSemantic = MemPool.CopyString(SrcElem.HLSLSemantic);
        }
        GraphicsPipeline.InputLayout.LayoutElements = pLayoutElements;
 
 
        // Correct description and compute offsets and tight strides
        std::array<Uint32, MAX_BUFFER_SLOTS> Strides, TightStrides = {};
        // Set all strides to an invalid value because an application may want to use 0 stride
        Strides.fill(LAYOUT_ELEMENT_AUTO_STRIDE);
 
        for (Uint32 i = 0; i < InputLayout.NumElements; ++i)
        {
            auto& LayoutElem = pLayoutElements[i];
 
            if (LayoutElem.ValueType == VT_FLOAT32 || LayoutElem.ValueType == VT_FLOAT16)
                LayoutElem.IsNormalized = false; // Floating point values cannot be normalized
 
            auto BuffSlot = LayoutElem.BufferSlot;
            if (BuffSlot >= Strides.size())
            {
                UNEXPECTED("Buffer slot (", BuffSlot, ") exceeds the maximum allowed value (", Strides.size() - 1, ")");
                continue;
            }
            BufferSlotsUsed = std::max(BufferSlotsUsed, static_cast<Uint8>(BuffSlot + 1));
 
            auto& CurrAutoStride = TightStrides[BuffSlot];
            // If offset is not explicitly specified, use current auto stride value
            if (LayoutElem.RelativeOffset == LAYOUT_ELEMENT_AUTO_OFFSET)
            {
                LayoutElem.RelativeOffset = CurrAutoStride;
            }
 
            // If stride is explicitly specified, use it for the current buffer slot
            if (LayoutElem.Stride != LAYOUT_ELEMENT_AUTO_STRIDE)
            {
                // Verify that the value is consistent with the previously specified stride, if any
                if (Strides[BuffSlot] != LAYOUT_ELEMENT_AUTO_STRIDE && Strides[BuffSlot] != LayoutElem.Stride)
                {
                    LOG_ERROR_MESSAGE("Inconsistent strides are specified for buffer slot ", BuffSlot,
                                      ". Input element at index ", LayoutElem.InputIndex, " explicitly specifies stride ",
                                      LayoutElem.Stride, ", while current value is ", Strides[BuffSlot],
                                      ". Specify consistent strides or use LAYOUT_ELEMENT_AUTO_STRIDE to allow "
                                      "the engine compute strides automatically.");
                }
                Strides[BuffSlot] = LayoutElem.Stride;
            }
 
            CurrAutoStride = std::max(CurrAutoStride, LayoutElem.RelativeOffset + LayoutElem.NumComponents * GetValueSize(LayoutElem.ValueType));
        }
 
        for (Uint32 i = 0; i < InputLayout.NumElements; ++i)
        {
            auto& LayoutElem = pLayoutElements[i];
 
            auto BuffSlot = LayoutElem.BufferSlot;
            // If no input elements explicitly specified stride for this buffer slot, use automatic stride
            if (Strides[BuffSlot] == LAYOUT_ELEMENT_AUTO_STRIDE)
            {
                Strides[BuffSlot] = TightStrides[BuffSlot];
            }
            else
            {
                if (Strides[BuffSlot] < TightStrides[BuffSlot])
                {
                    LOG_ERROR_MESSAGE("Stride ", Strides[BuffSlot], " explicitly specified for slot ", BuffSlot,
                                      " is smaller than the minimum stride ", TightStrides[BuffSlot],
                                      " required to accomodate all input elements.");
                }
            }
            if (LayoutElem.Stride == LAYOUT_ELEMENT_AUTO_STRIDE)
                LayoutElem.Stride = Strides[BuffSlot];
        }
 
        pStrides = MemPool.ConstructArray<Uint32>(BufferSlotsUsed);
 
        // Set strides for all unused slots to 0
        for (Uint32 i = 0; i < BufferSlotsUsed; ++i)
        {
            auto Stride = Strides[i];
            pStrides[i] = Stride != LAYOUT_ELEMENT_AUTO_STRIDE ? Stride : 0;
        }
    }
 
    void InitializePipelineDesc(const ComputePipelineStateCreateInfo& CreateInfo,
                                FixedLinearAllocator&                 MemPool)
    {
        m_pPipelineDataRawMem = MemPool.ReleaseOwnership();
 
        CopyResourceLayout(CreateInfo.PSODesc.ResourceLayout, this->m_Desc.ResourceLayout, MemPool);
        CopyResourceSignatures(CreateInfo, MemPool);
    }
 
    void InitializePipelineDesc(const RayTracingPipelineStateCreateInfo& CreateInfo,
                                FixedLinearAllocator&                    MemPool) noexcept
    {
        size_t RTDataSize = sizeof(RayTracingPipelineData);
        // Allocate space for shader handles
        const auto ShaderHandleSize = this->m_pDevice->GetProperties().ShaderGroupHandleSize;
        const auto ShaderDataSize   = ShaderHandleSize * (CreateInfo.GeneralShaderCount + CreateInfo.TriangleHitShaderCount + CreateInfo.ProceduralHitShaderCount);
        RTDataSize += ShaderDataSize;
        // Extra bytes were reserved to avoid compiler errors on zero-sized arrays
        RTDataSize -= sizeof(RayTracingPipelineData::ShaderHandles);
 
        this->m_pRayTracingPipelineData = static_cast<RayTracingPipelineData*>(MemPool.Allocate(RTDataSize, alignof(RayTracingPipelineData)));
        new (this->m_pRayTracingPipelineData) RayTracingPipelineData{};
        this->m_pRayTracingPipelineData->ShaderHandleSize = ShaderHandleSize;
        this->m_pRayTracingPipelineData->Desc             = CreateInfo.RayTracingPipeline;
        this->m_pRayTracingPipelineData->ShaderDataSize   = ShaderDataSize;
 
        void* Ptr = MemPool.ReleaseOwnership();
        VERIFY_EXPR(Ptr == m_pPipelineDataRawMem);
 
        TNameToGroupIndexMap& NameToGroupIndex = this->m_pRayTracingPipelineData->NameToGroupIndex;
        CopyRTShaderGroupNames(NameToGroupIndex, CreateInfo, MemPool);
 
        CopyResourceLayout(CreateInfo.PSODesc.ResourceLayout, this->m_Desc.ResourceLayout, MemPool);
        CopyResourceSignatures(CreateInfo, MemPool);
    }
 
 
    // Resource attribution properties
    struct ResourceAttribution
    {
        static constexpr Uint32 InvalidSignatureIndex = ~0u;
        static constexpr Uint32 InvalidResourceIndex  = PipelineResourceSignatureImplType::InvalidResourceIndex;
        static constexpr Uint32 InvalidSamplerIndex   = InvalidImmutableSamplerIndex;
 
        const PipelineResourceSignatureImplType* pSignature = nullptr;
 
        Uint32 SignatureIndex        = InvalidSignatureIndex;
        Uint32 ResourceIndex         = InvalidResourceIndex;
        Uint32 ImmutableSamplerIndex = InvalidSamplerIndex;
 
        ResourceAttribution() noexcept {}
        ResourceAttribution(const PipelineResourceSignatureImplType* _pSignature,
                            Uint32                                   _SignatureIndex,
                            Uint32                                   _ResourceIndex,
                            Uint32                                   _ImmutableSamplerIndex = InvalidResourceIndex) noexcept :
            pSignature{_pSignature},
            SignatureIndex{_SignatureIndex},
            ResourceIndex{_ResourceIndex},
            ImmutableSamplerIndex{_ImmutableSamplerIndex}
        {
            VERIFY_EXPR(pSignature == nullptr || pSignature->GetDesc().BindingIndex == SignatureIndex);
            VERIFY_EXPR((ResourceIndex == InvalidResourceIndex) || (ImmutableSamplerIndex == InvalidSamplerIndex));
        }
 
        explicit operator bool() const
        {
            return ((SignatureIndex != InvalidSignatureIndex) &&
                    (ResourceIndex != InvalidResourceIndex || ImmutableSamplerIndex != InvalidSamplerIndex));
        }
 
        bool IsImmutableSampler() const
        {
            return operator bool() && ImmutableSamplerIndex != InvalidSamplerIndex;
        }
    };
 
    ResourceAttribution GetResourceAttribution(const char* Name, SHADER_TYPE Stage) const
    {
        const auto* const pThis = static_cast<const PipelineStateImplType*>(this);
 
        const auto SignCount = pThis->GetResourceSignatureCount();
        for (Uint32 sign = 0; sign < SignCount; ++sign)
        {
            const PipelineResourceSignatureImplType* const pSignature = pThis->GetResourceSignature(sign);
            if (pSignature == nullptr)
                continue;
 
            const auto ResIndex = pSignature->FindResource(Stage, Name);
            if (ResIndex != ResourceAttribution::InvalidResourceIndex)
                return ResourceAttribution{pSignature, sign, ResIndex};
            else
            {
                const auto ImtblSamIndex = pSignature->FindImmutableSampler(Stage, Name);
                if (ImtblSamIndex != ResourceAttribution::InvalidSamplerIndex)
                    return ResourceAttribution{pSignature, sign, ResourceAttribution::InvalidResourceIndex, ImtblSamIndex};
            }
        }
        return ResourceAttribution{};
    }
 
private:
    static void ReserveResourceLayout(const PipelineResourceLayoutDesc& SrcLayout, FixedLinearAllocator& MemPool) noexcept
    {
        if (SrcLayout.Variables != nullptr)
        {
            MemPool.AddSpace<ShaderResourceVariableDesc>(SrcLayout.NumVariables);
            for (Uint32 i = 0; i < SrcLayout.NumVariables; ++i)
            {
                VERIFY(SrcLayout.Variables[i].Name != nullptr, "Variable name can't be null");
                MemPool.AddSpaceForString(SrcLayout.Variables[i].Name);
            }
        }
 
        if (SrcLayout.ImmutableSamplers != nullptr)
        {
            MemPool.AddSpace<ImmutableSamplerDesc>(SrcLayout.NumImmutableSamplers);
            for (Uint32 i = 0; i < SrcLayout.NumImmutableSamplers; ++i)
            {
                VERIFY(SrcLayout.ImmutableSamplers[i].SamplerOrTextureName != nullptr, "Immutable sampler or texture name can't be null");
                MemPool.AddSpaceForString(SrcLayout.ImmutableSamplers[i].SamplerOrTextureName);
            }
        }
 
        static_assert(std::is_trivially_destructible<decltype(*SrcLayout.Variables)>::value, "Add destructor for this object to Destruct()");
        static_assert(std::is_trivially_destructible<decltype(*SrcLayout.ImmutableSamplers)>::value, "Add destructor for this object to Destruct()");
    }
 
    static void CopyResourceLayout(const PipelineResourceLayoutDesc& SrcLayout, PipelineResourceLayoutDesc& DstLayout, FixedLinearAllocator& MemPool)
    {
        if (SrcLayout.Variables != nullptr)
        {
            auto* const Variables = MemPool.ConstructArray<ShaderResourceVariableDesc>(SrcLayout.NumVariables);
            DstLayout.Variables   = Variables;
            for (Uint32 i = 0; i < SrcLayout.NumVariables; ++i)
            {
                const auto& SrcVar = SrcLayout.Variables[i];
                Variables[i]       = SrcVar;
                Variables[i].Name  = MemPool.CopyString(SrcVar.Name);
            }
        }
 
        if (SrcLayout.ImmutableSamplers != nullptr)
        {
            auto* const ImmutableSamplers = MemPool.ConstructArray<ImmutableSamplerDesc>(SrcLayout.NumImmutableSamplers);
            DstLayout.ImmutableSamplers   = ImmutableSamplers;
            for (Uint32 i = 0; i < SrcLayout.NumImmutableSamplers; ++i)
            {
                const auto& SrcSmplr = SrcLayout.ImmutableSamplers[i];
#ifdef DILIGENT_DEVELOPMENT
                {
                    const auto& BorderColor = SrcSmplr.Desc.BorderColor;
                    if (!((BorderColor[0] == 0 && BorderColor[1] == 0 && BorderColor[2] == 0 && BorderColor[3] == 0) ||
                          (BorderColor[0] == 0 && BorderColor[1] == 0 && BorderColor[2] == 0 && BorderColor[3] == 1) ||
                          (BorderColor[0] == 1 && BorderColor[1] == 1 && BorderColor[2] == 1 && BorderColor[3] == 1)))
                    {
                        LOG_WARNING_MESSAGE("Immutable sampler for variable \"", SrcSmplr.SamplerOrTextureName, "\" specifies border color (",
                                            BorderColor[0], ", ", BorderColor[1], ", ", BorderColor[2], ", ", BorderColor[3],
                                            "). D3D12 static samplers only allow transparent black (0,0,0,0), opaque black (0,0,0,1) or opaque white (1,1,1,1) as border colors");
                    }
                }
#endif
 
                ImmutableSamplers[i]                      = SrcSmplr;
                ImmutableSamplers[i].SamplerOrTextureName = MemPool.CopyString(SrcSmplr.SamplerOrTextureName);
            }
        }
    }
 
    void ReserveResourceSignatures(const PipelineStateCreateInfo& CreateInfo, FixedLinearAllocator& MemPool)
    {
        if (m_UsingImplicitSignature)
        {
            VERIFY_EXPR(CreateInfo.ResourceSignaturesCount == 0 || CreateInfo.ppResourceSignatures == nullptr);
            m_SignatureCount = 1;
        }
        else
        {
            VERIFY_EXPR(CreateInfo.ResourceSignaturesCount > 0 && CreateInfo.ppResourceSignatures != nullptr);
            Uint32 MaxSignatureBindingIndex = 0;
            for (Uint32 i = 0; i < CreateInfo.ResourceSignaturesCount; ++i)
            {
                const auto* pSignature = ValidatedCast<PipelineResourceSignatureImplType>(CreateInfo.ppResourceSignatures[i]);
                VERIFY(pSignature != nullptr, "Pipeline resource signature at index ", i, " is null. This error should've been caught by ValidatePipelineResourceSignatures.");
 
                Uint32 Index = pSignature->GetDesc().BindingIndex;
                VERIFY(Index < MAX_RESOURCE_SIGNATURES,
                       "Pipeline resource signature specifies binding index ", Uint32{Index}, " that exceeds the limit (", MAX_RESOURCE_SIGNATURES - 1,
                       "). This error should've been caught by ValidatePipelineResourceSignatureDesc.");
 
                MaxSignatureBindingIndex = std::max(MaxSignatureBindingIndex, Uint32{Index});
            }
            VERIFY_EXPR(MaxSignatureBindingIndex < MAX_RESOURCE_SIGNATURES);
            m_SignatureCount = static_cast<decltype(m_SignatureCount)>(MaxSignatureBindingIndex + 1);
            VERIFY_EXPR(m_SignatureCount == MaxSignatureBindingIndex + 1);
        }
 
        MemPool.AddSpace<SignatureAutoPtrType>(m_SignatureCount);
    }
 
    void CopyResourceSignatures(const PipelineStateCreateInfo& CreateInfo, FixedLinearAllocator& MemPool)
    {
        m_Signatures = MemPool.ConstructArray<SignatureAutoPtrType>(m_SignatureCount);
        if (!m_UsingImplicitSignature)
        {
            VERIFY_EXPR(CreateInfo.ResourceSignaturesCount != 0 && CreateInfo.ppResourceSignatures != nullptr);
            for (Uint32 i = 0; i < CreateInfo.ResourceSignaturesCount; ++i)
            {
                auto* pSignature = ValidatedCast<PipelineResourceSignatureImplType>(CreateInfo.ppResourceSignatures[i]);
                VERIFY_EXPR(pSignature != nullptr);
 
                const Uint32 Index = pSignature->GetDesc().BindingIndex;
 
#ifdef DILIGENT_DEBUG
                VERIFY_EXPR(Index < m_SignatureCount);
 
                VERIFY(m_Signatures[Index] == nullptr,
                       "Pipeline resource signature '", pSignature->GetDesc().Name, "' at index ", Uint32{Index},
                       " conflicts with another resource signature '", m_Signatures[Index]->GetDesc().Name,
                       "' that uses the same index. This error should've been caught by ValidatePipelineResourceSignatures.");
 
                for (Uint32 s = 0, StageCount = pSignature->GetNumActiveShaderStages(); s < StageCount; ++s)
                {
                    const auto ShaderType = pSignature->GetActiveShaderStageType(s);
                    VERIFY(IsConsistentShaderType(ShaderType, CreateInfo.PSODesc.PipelineType),
                           "Pipeline resource signature '", pSignature->GetDesc().Name, "' at index ", Uint32{Index},
                           " has shader stage '", GetShaderTypeLiteralName(ShaderType), "' that is not compatible with pipeline type '",
                           GetPipelineTypeString(CreateInfo.PSODesc.PipelineType), "'.");
                }
#endif
 
                m_Signatures[Index] = pSignature;
            }
        }
    }
 
protected:
    SHADER_TYPE m_ActiveShaderStages = SHADER_TYPE_UNKNOWN;
 
    const bool m_UsingImplicitSignature;
 
    Uint8 m_SignatureCount = 0;
 
    using SignatureAutoPtrType         = RefCntAutoPtr<PipelineResourceSignatureImplType>;
    SignatureAutoPtrType* m_Signatures = nullptr; // [m_SignatureCount]
 
    struct GraphicsPipelineData
    {
        GraphicsPipelineDesc Desc;
 
        RefCntAutoPtr<IRenderPass> pRenderPass; 
 
        Uint32* pStrides        = nullptr;
        Uint8   BufferSlotsUsed = 0;
    };
 
    struct RayTracingPipelineData
    {
        RayTracingPipelineDesc Desc;
 
        // Mapping from the shader group name to its index in the pipeline.
        // It is used to find the shader handle in ShaderHandles array.
        TNameToGroupIndexMap NameToGroupIndex;
 
        Uint32 ShaderHandleSize = 0;
        Uint32 ShaderDataSize   = 0;
 
        // Array of shader handles for every group in the pipeline.
        // The handles will be copied to the SBT using NameToGroupIndex to find
        // handles by group name.
        // The actual array size will be determined at run time and will be stored
        // in ShaderDataSize.
        Uint8 ShaderHandles[sizeof(void*)] = {};
    };
    static_assert(offsetof(RayTracingPipelineData, ShaderHandles) % sizeof(void*) == 0, "ShaderHandles member is expected to be sizeof(void*)-aligned");
 
    union
    {
        GraphicsPipelineData*   m_pGraphicsPipelineData;
        RayTracingPipelineData* m_pRayTracingPipelineData;
        void*                   m_pPipelineDataRawMem = nullptr;
    };
 
#ifdef DILIGENT_DEBUG
    bool m_IsDestructed = false;
#endif
};
 
} // namespace Diligent