/*
 * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */
#include "containers/map.h"
#include "containers/multimap.h"
#include "nvctassert.h"
#include "nvmisc.h"
#include "nvport/sync.h"
#include "nvrm_registry.h"
#include "os/os.h"
#include "rmapi/control.h"
#include "rmapi/rmapi.h"
#include "rmapi/rmapi_utils.h"
#include "ctrl/ctrl0000/ctrl0000system.h"
#include "ctrl/ctrl2080/ctrl2080gpu.h"
#include "ctrl/ctrl2080/ctrl2080fifo.h"
#include "ctrl/ctrl2080/ctrl2080bus.h"

typedef struct
{
    void* params;
} RmapiControlCacheEntry;

//
// Stores the mapping of client object to the corresponding GPU instance
// The key is generated by _handlesToGpuInstKey with client and object handle
// The value is the instance of the GPU (pGpu->gpuInstance) the object is linked to
//
MAKE_MAP(ObjectToGpuInstMap, NvU64);

//
// Stores the cached control value.
// Each submap in the multimap stores the cached control value for one GPU.
// The key to find a submap is GPU Instance stored in ObjectToGpuInstMap
//
// The key inside the submap is the control command
// The value inside the submap is the cached control value for the command
//
MAKE_MULTIMAP(GpusControlCache, RmapiControlCacheEntry);

ct_assert(sizeof(NvHandle) <= 4);

#define CLIENT_KEY_SHIFT (sizeof(NvHandle) * 8)

static NvBool _isCmdSystemWide(NvU32 cmd)
{
    return DRF_VAL(XXXX, _CTRL_CMD, _CLASS, cmd) == NV01_ROOT;
}

static NvHandle _gpuInstKeyToClient(NvU64 key)
{
    return (key >> CLIENT_KEY_SHIFT);
}

static NvU64 _handlesToGpuInstKey(NvHandle hClient, NvHandle hObject)
{
    return ((NvU64)hClient << CLIENT_KEY_SHIFT) | hObject;
}

static struct {
    /* NOTE: Size unbounded for now */
    GpusControlCache gpusControlCache;
    ObjectToGpuInstMap objectToGpuInstMap;
    NvU32 mode;
    PORT_RWLOCK *pLock;
} RmapiControlCache;

enum CACHE_LOCK_TYPE
{
    LOCK_EXCLUSIVE,
    LOCK_SHARED
};

static void _cacheLockAcquire(enum CACHE_LOCK_TYPE lockType)
{
    if (lockType == LOCK_EXCLUSIVE)
        portSyncRwLockAcquireWrite(RmapiControlCache.pLock);
    else
        portSyncRwLockAcquireRead(RmapiControlCache.pLock);
}

static void _cacheLockRelease(enum CACHE_LOCK_TYPE lockType)
{
    if (lockType == LOCK_EXCLUSIVE)
        portSyncRwLockReleaseWrite(RmapiControlCache.pLock);
    else
        portSyncRwLockReleaseRead(RmapiControlCache.pLock);
}

static RmapiControlCacheEntry* _getOrInitCacheEntry(NvU64 key1, NvU64 key2, NvBool bSet,
                                                    NvU32 allocSize, NvBool *pbParamAllocated);

NvBool rmapiControlIsCacheable(NvU32 flags, NvU32 accessRight, NvBool bAllowInternal)
{
    if (RmapiControlCache.mode == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_DISABLE)
        return NV_FALSE;

    if (!(flags & RMCTRL_FLAGS_CACHEABLE_ANY))
        return NV_FALSE;

    //
    // RMCTRL with access right requires access right check.
    // We need resource ref and client object to check the access right, both not
    // available here. Do not cache RMCTRLs with access right.
    //
    if (accessRight != 0)
        return NV_FALSE;

    // Allow internal RMCTRL of all privilege with bAllowInternal flag
    if (flags & RMCTRL_FLAGS_INTERNAL)
        return bAllowInternal;

    //
    // For non-internal calls, allow cache only for non-privileged RMCTRL since
    // we don't have the client object to validate client.
    //
    if (!(flags & RMCTRL_FLAGS_NON_PRIVILEGED))
    {
        NV_PRINTF(LEVEL_WARNING, "Do not allow cache for priviliged ctrls\n");
        return NV_FALSE;
    }

    return NV_TRUE;
}

NvBool rmapiCmdIsCacheable(NvU32 cmd, NvBool bAllowInternal)
{
    NvU32 flags;
    NvU32 accessRight;

    if (rmapiutilGetControlInfo(cmd, &flags, &accessRight) != NV_OK)
        return NV_FALSE;

    return rmapiControlIsCacheable(flags, accessRight, bAllowInternal);
}

NV_STATUS rmapiControlCacheInit(void)
{
#if defined(DEBUG)
    RmapiControlCache.mode = NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_VERIFY_ONLY;
#else
    RmapiControlCache.mode = NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_ENABLE;
#endif

    NvU32 mode;

    if (osReadRegistryDword(NULL, NV_REG_STR_RM_CACHEABLE_CONTROLS, &mode) == NV_OK)
    {
        RmapiControlCache.mode = mode;
    }
    NV_PRINTF(LEVEL_INFO, "using cache mode %d\n", RmapiControlCache.mode);

    multimapInit(&RmapiControlCache.gpusControlCache, portMemAllocatorGetGlobalNonPaged());
    mapInit(&RmapiControlCache.objectToGpuInstMap, portMemAllocatorGetGlobalNonPaged());
    RmapiControlCache.pLock = portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
    if (RmapiControlCache.pLock == NULL)
    {
        NV_PRINTF(LEVEL_ERROR, "failed to create rw lock\n");
        multimapDestroy(&RmapiControlCache.gpusControlCache);
        mapDestroy(&RmapiControlCache.objectToGpuInstMap);
        return NV_ERR_NO_MEMORY;
    }
    return NV_OK;
}

NV_STATUS rmapiControlCacheSetGpuInstForObject
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 gpuInst
)
{
    NV_STATUS status = NV_OK;
    NvU64 *entry;

    if (gpuInst >= NV_MAX_DEVICES)
        return NV_ERR_INVALID_PARAMETER;

    NV_PRINTF(LEVEL_INFO, "gpu inst set for 0x%x 0x%x: 0x%x\n", hClient, hObject, gpuInst);

    _cacheLockAcquire(LOCK_EXCLUSIVE);
    entry = mapFind(&RmapiControlCache.objectToGpuInstMap, _handlesToGpuInstKey(hClient, hObject));

    if (entry != NULL)
    {
        // set happens in object allocation, should not exist in cache already
        NV_PRINTF(LEVEL_WARNING,
                  "set existing gpu inst 0x%x 0x%x was 0x%llx is 0x%x\n",
                  hClient, hObject, *entry, gpuInst);
        status = NV_ERR_INVALID_PARAMETER;
        goto done;
    }

    entry = mapInsertNew(&RmapiControlCache.objectToGpuInstMap, _handlesToGpuInstKey(hClient, hObject));

    if (entry == NULL)
    {
        status = NV_ERR_NO_MEMORY;
        goto done;
    }

    *entry = gpuInst;

done:
    _cacheLockRelease(LOCK_EXCLUSIVE);
    return status;
}

// Need to hold rmapi control cache read/write lock
static NV_STATUS _rmapiControlCacheGetGpuInstForObject
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 *pGpuInst
)
{
    NV_STATUS status = NV_ERR_OBJECT_NOT_FOUND;
    NvU64* entry = mapFind(&RmapiControlCache.objectToGpuInstMap, _handlesToGpuInstKey(hClient, hObject));

    NV_PRINTF(LEVEL_INFO, "cached gpu inst lookup for 0x%x 0x%x\n", hClient, hObject);

    if (entry != NULL)
    {
        NV_PRINTF(LEVEL_INFO, "cached gpu inst for 0x%x 0x%x: 0x%llx\n", hClient, hObject, *entry);
        *pGpuInst = *entry;
        status = NV_OK;
    }

    return status;
}

// Need to hold rmapi control cache write lock
static void _rmapiControlCacheFreeGpuInstForObject
(
    NvHandle hClient,
    NvHandle hObject
)
{
    const NvU64 key = _handlesToGpuInstKey(hClient, hObject);
    NvU64* entry = mapFind(&RmapiControlCache.objectToGpuInstMap, key);

    if (entry != NULL)
    {
        mapRemove(&RmapiControlCache.objectToGpuInstMap, entry);
        NV_PRINTF(LEVEL_INFO, "Gpu Inst entry with key 0x%llx freed\n", key);
    }
}

// Need to hold rmapi control cache write lock
static void _rmapiControlCacheFreeGpuInstForClient(NvHandle hClient)
{
    while (NV_TRUE)
    {
        NvU64* entry = mapFindGEQ(&RmapiControlCache.objectToGpuInstMap, _handlesToGpuInstKey(hClient, 0));
        NvU64 key;

        if (entry == NULL)
            break;

        key = mapKey(&RmapiControlCache.objectToGpuInstMap, entry);

        if (_gpuInstKeyToClient(key) != hClient)
            break;

        mapRemove(&RmapiControlCache.objectToGpuInstMap, entry);
        NV_PRINTF(LEVEL_INFO, "Gpu Inst entry with key 0x%llx freed\n", key);
    }
}

static NV_STATUS _rmapiControlCacheGet
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 cmd,
    void* params,
    NvU32 paramsSize
)
{
    RmapiControlCacheEntry *entry;
    NvU32 gpuInst;
    NV_STATUS status = NV_OK;

    _cacheLockAcquire(LOCK_SHARED);

    if (rmapiControlCacheGetMode() == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_DISABLE)
    {
        // unexpected mode change.
        status = NV_ERR_INVALID_STATE;
        goto done;
    }

    if (_isCmdSystemWide(cmd))
    {
       gpuInst = NV_MAX_DEVICES;
    }
    else
    {
        status = _rmapiControlCacheGetGpuInstForObject(hClient, hObject, &gpuInst);
        if (status != NV_OK)
            goto done;
    }

    entry = _getOrInitCacheEntry(gpuInst, cmd, NV_FALSE, 0, NULL);
    if (entry == NULL || entry->params == NULL)
    {
        status = NV_ERR_OBJECT_NOT_FOUND;
        goto done;
    }

    portMemCopy(params, paramsSize, entry->params, paramsSize);
done:
    _cacheLockRelease(LOCK_SHARED);
    return status;
}

static NV_STATUS _rmapiControlCacheSet
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 cmd,
    const void* params,
    NvU32 paramsSize
)
{
    NV_STATUS status = NV_OK;
    RmapiControlCacheEntry* entry = NULL;
    NvU32 gpuInst;
    NvBool bParamsAllocated;

    _cacheLockAcquire(LOCK_EXCLUSIVE);

    if (rmapiControlCacheGetMode() == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_DISABLE)
    {
        // unexpected mode change.
        status = NV_ERR_INVALID_STATE;
        goto done;
    }

    if (_isCmdSystemWide(cmd))
    {
        gpuInst = NV_MAX_DEVICES;
    }
    else
    {
        status = _rmapiControlCacheGetGpuInstForObject(hClient, hObject, &gpuInst);
        if (status != NV_OK)
            goto done;
    }

    entry = _getOrInitCacheEntry(gpuInst, cmd, NV_TRUE, paramsSize, &bParamsAllocated);
    if (entry == NULL)
    {
        status = NV_ERR_NO_MEMORY;
        goto done;
    }

    //
    // A succeeded getOrInit call without params allocated implies
    // duplicated cache insertion that should be skipped.
    // Duplicated cache set happens when
    // 1. Parallel controls call into RM before first cache set.
    //    All threads will attempt cache set after the control calls.
    // 2. Cache already set by RPC to GSP path
    // 3. Cache in verify only mode
    //
    if (!bParamsAllocated)
    {
        if (RmapiControlCache.mode == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_VERIFY_ONLY)
        {
            NV_ASSERT(portMemCmp(entry->params, params, paramsSize) == 0);
        }
        status = NV_OK;
        goto done;
    }

    portMemCopy(entry->params, paramsSize, params, paramsSize);

done:
    _cacheLockRelease(LOCK_EXCLUSIVE);
    return status;
}

//
// Common function to find existing or generate new cache entry
// For cache get, we return the cache map query result directly
// For cache set, we generate new cache entry if there's no existing entry
// in the map.
//
//   key1 [IN]
//     First key for the multimap entry
//   key2 [IN]
//     Second key for the multimap entry
//   bSet [IN]
//     If the query is for cache set or cache get
//   allocSize [IN]
//     For cache set only.
//     The size to allocate for new cache entry
//   pbParamsAllocated [OUT]
//     For cache set only.
//     Indicate if we allocated new memory for cache entry
//     A cache set without memory allocation implies there's an existng entry.
//
static RmapiControlCacheEntry*
_getOrInitCacheEntry
(
    NvU64 key1,
    NvU64 key2,
    NvBool bSet,
    NvU32 allocSize,
    NvBool *pbParamsAllocated
)
{
    RmapiControlCacheEntry *entry = NULL;
    GpusControlCacheSubmap *insertedSubmap = NULL;

    entry = multimapFindItem(&RmapiControlCache.gpusControlCache, key1, key2);

    // for cache get, return map find result directly
    if (!bSet)
        return entry;

    // for cache set, try to init entry if not valid
    if (entry == NULL)
    {
        if (multimapFindSubmap(&RmapiControlCache.gpusControlCache, key1) == NULL)
        {
            insertedSubmap = multimapInsertSubmap(&RmapiControlCache.gpusControlCache, key1);
            if (insertedSubmap == NULL)
                goto failed;
        }

        entry = multimapInsertItemNew(&RmapiControlCache.gpusControlCache, key1, key2);
    }

    if (entry == NULL)
        goto failed_free_submap;

    if (entry->params == NULL)
    {
        entry->params = portMemAllocNonPaged(allocSize);
        if (entry->params == NULL)
            goto failed_free_entry;

        portMemSet(entry->params, 0, allocSize);

        if (pbParamsAllocated != NULL)
            *pbParamsAllocated = NV_TRUE;
    }
    else if (pbParamsAllocated != NULL)
    {
        *pbParamsAllocated = NV_FALSE;
    }

    return entry;

failed_free_entry:
    if (entry != NULL)
        multimapRemoveItem(&RmapiControlCache.gpusControlCache, entry);
failed_free_submap:
    if (insertedSubmap != NULL)
        multimapRemoveSubmap(&RmapiControlCache.gpusControlCache, insertedSubmap);
failed:
    return NULL;
}

static NvBool _isGpuGetInfoIndexCacheable(NvU32 index)
{
    switch (index)
    {
        case NV2080_CTRL_GPU_INFO_INDEX_MINOR_REVISION_EXT:
        case NV2080_CTRL_GPU_INFO_INDEX_NETLIST_REV0:
        case NV2080_CTRL_GPU_INFO_INDEX_NETLIST_REV1:
        case NV2080_CTRL_GPU_INFO_INDEX_SYSMEM_ACCESS:
        case NV2080_CTRL_GPU_INFO_INDEX_GEMINI_BOARD:
        case NV2080_CTRL_GPU_INFO_INDEX_SURPRISE_REMOVAL_POSSIBLE:
        case NV2080_CTRL_GPU_INFO_INDEX_GLOBAL_POISON_FUSE_ENABLED:
        case NV2080_CTRL_GPU_INFO_INDEX_GPU_SR_SUPPORT:
        case NV2080_CTRL_GPU_INFO_INDEX_SPLIT_VAS_MGMT_SERVER_CLIENT_RM:
        case NV2080_CTRL_GPU_INFO_INDEX_GPU_SM_VERSION:
        case NV2080_CTRL_GPU_INFO_INDEX_4K_PAGE_ISOLATION_REQUIRED:
        case NV2080_CTRL_GPU_INFO_INDEX_DISPLAY_ENABLED:
        case NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED:
        case NV2080_CTRL_GPU_INFO_INDEX_GPU_PROFILING_CAPABILITY:
        case NV2080_CTRL_GPU_INFO_INDEX_GPU_DEBUGGING_CAPABILITY:
        case NV2080_CTRL_GPU_INFO_INDEX_CMP_SKU:
        case NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY:
            return NV_TRUE;
        default:
            return NV_FALSE;
    }
}

static NvBool _isFifoGetInfoIndexCacheable(NvU32 index)
{
    switch (index)
    {
        case NV2080_CTRL_FIFO_INFO_INDEX_INSTANCE_TOTAL:
        case NV2080_CTRL_FIFO_INFO_INDEX_MAX_CHANNEL_GROUPS:
        case NV2080_CTRL_FIFO_INFO_INDEX_MAX_CHANNELS_PER_GROUP:
        case NV2080_CTRL_FIFO_INFO_INDEX_MAX_SUBCONTEXT_PER_GROUP:
        case NV2080_CTRL_FIFO_INFO_INDEX_BAR1_USERD_START_OFFSET:
        case NV2080_CTRL_FIFO_INFO_INDEX_DEFAULT_CHANNEL_TIMESLICE:
            return NV_TRUE;
        default:
            return NV_FALSE;
    }
}

static NvBool _isBusGetInfoIndexCacheable(NvU32 index)
{
    switch (index)
    {
        case NV2080_CTRL_BUS_INFO_INDEX_TYPE:
        case NV2080_CTRL_BUS_INFO_INDEX_INTLINE:
        case NV2080_CTRL_BUS_INFO_INDEX_CAPS:
        case NV2080_CTRL_BUS_INFO_INDEX_PCIE_DOWNSTREAM_LINK_CAPS:
        case NV2080_CTRL_BUS_INFO_INDEX_COHERENT_DMA_FLAGS:
        case NV2080_CTRL_BUS_INFO_INDEX_NONCOHERENT_DMA_FLAGS:
        case NV2080_CTRL_BUS_INFO_INDEX_BUS_NUMBER:
        case NV2080_CTRL_BUS_INFO_INDEX_DEVICE_NUMBER:
        case NV2080_CTRL_BUS_INFO_INDEX_DOMAIN_NUMBER:
        case NV2080_CTRL_BUS_INFO_INDEX_INTERFACE_TYPE:
        case NV2080_CTRL_BUS_INFO_INDEX_GPU_INTERFACE_TYPE:
            return NV_TRUE;
        default:
            return NV_FALSE;
    }
}

static NvBool _isGetInfoIndexCacheable(NvU32 cmd, NvU32 index)
{
    switch (cmd)
    {
        case NV2080_CTRL_CMD_GPU_GET_INFO_V2:
            return _isGpuGetInfoIndexCacheable(index);
        case NV2080_CTRL_CMD_FIFO_GET_INFO:
            return _isFifoGetInfoIndexCacheable(index);
        case NV2080_CTRL_CMD_BUS_GET_INFO_V2:
            return _isBusGetInfoIndexCacheable(index);
    }

    return NV_FALSE;
}

//
// For GET_INFO controls, we use an array of getInfoCacheEntry to store the
// cached value.
//
// The length of the array is the max list length of each control and is
// enough to store the cached value of all indexes.
//
// The Nth item in the array, array[N], represent the cache state of the info
// whose index value is N. If the info is cached, array[N].valid is NV_TRUE
// and the cached value is stored in array[N].data.
// array[N].valid is NV_FALSE if the info is not cached.
//
typedef struct GetInfoCacheEntry {
    NvBool valid;
    NvU32 data;
} GetInfoCacheEntry;

static NV_STATUS _getInfoCacheHandler
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 cmd,
    NVXXXX_CTRL_XXX_INFO *pInfo,
    NvU32 listSize,
    NvU32 listSizeLimit,
    NvBool bSet
)
{
    NV_STATUS status = NV_OK;
    NvU32 i = 0;
    NvU32 gpuInst;
    RmapiControlCacheEntry *entry = NULL;
    GetInfoCacheEntry *cachedTable = NULL;
    const NvU32 allocSize = sizeof(GetInfoCacheEntry) * listSizeLimit;
    enum CACHE_LOCK_TYPE lockType = bSet ? LOCK_EXCLUSIVE : LOCK_SHARED;

    if (listSize <= 0 || listSize > listSizeLimit || pInfo == NULL)
    {
        return NV_ERR_INVALID_PARAMETER;
    }

    _cacheLockAcquire(lockType);

    if (rmapiControlCacheGetMode() == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_DISABLE)
    {
        // unexpected mode change.
        status = NV_ERR_INVALID_STATE;
        goto done;
    }

    status = _rmapiControlCacheGetGpuInstForObject(hClient, hObject, &gpuInst);
    if (status != NV_OK)
        goto done;

    entry = _getOrInitCacheEntry(gpuInst, cmd, bSet, allocSize, NULL);

    if (entry == NULL || entry->params == NULL)
    {
        status = bSet ? NV_ERR_NO_MEMORY : NV_ERR_OBJECT_NOT_FOUND;
        goto done;
    }

    cachedTable = (GetInfoCacheEntry*)entry->params;

    for (i = 0; i < listSize; ++i)
    {
        const NvU32 index = pInfo[i].index;

        if (index >= listSizeLimit)
        {
            status = NV_ERR_INVALID_ARGUMENT;
            goto done;
        }

        if (bSet)
        {
            if (_isGetInfoIndexCacheable(cmd, index))
            {
                if (cachedTable[index].valid)
                {
                    NV_ASSERT(cachedTable[index].data == pInfo[i].data);
                }
                else
                {
                    cachedTable[index].valid = NV_TRUE;
                    cachedTable[index].data = pInfo[i].data;
                }
            }
        }
        else
        {
            // if any of the entry is not cacheable or not in the cache, skip the whole cmd
            if (!_isGetInfoIndexCacheable(cmd, index) || !cachedTable[index].valid)
            {
                status = NV_ERR_OBJECT_NOT_FOUND;
                goto done;
            }
        }
    }

    if (!bSet)
    {
        for (i = 0; i < listSize; ++i)
            pInfo[i].data = cachedTable[pInfo[i].index].data;
    }

done:

    if (status != NV_OK && bSet)
    {
        if (entry != NULL)
        {
            portMemFree(entry->params);
            multimapRemoveItem(&RmapiControlCache.gpusControlCache, entry);
        }
    }

    _cacheLockRelease(lockType);

    return status;
}

typedef struct GpuNameStringCacheEntry
{
    NvBool bAsciiValid;
    NvBool bUnicodeValid;
    NvU8  ascii[NV2080_GPU_MAX_NAME_STRING_LENGTH];
    NvU16 unicode[NV2080_GPU_MAX_NAME_STRING_LENGTH];
} GpuNameStringCacheEntry;

NV_STATUS _gpuNameStringGet
(
    NvHandle hClient,
    NvHandle hObject,
    NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS *pParams
)
{
    NvU32 gpuInst;
    NV_STATUS status = NV_OK;
    RmapiControlCacheEntry *entry = NULL;
    GpuNameStringCacheEntry *cachedParams = NULL;

    _cacheLockAcquire(LOCK_SHARED);

    if (rmapiControlCacheGetMode() == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_DISABLE)
    {
        // unexpected mode change.
        status = NV_ERR_INVALID_STATE;
        goto done;
    }

    status = _rmapiControlCacheGetGpuInstForObject(hClient, hObject, &gpuInst);
    if (status != NV_OK)
        goto done;

    entry = _getOrInitCacheEntry(gpuInst, NV2080_CTRL_CMD_GPU_GET_NAME_STRING,
                                 NV_FALSE, 0, NULL);
    if (entry == NULL || entry->params == NULL)
    {
        status = NV_ERR_OBJECT_NOT_FOUND;
        goto done;
    }

    cachedParams = (GpuNameStringCacheEntry *)entry->params;

    switch (pParams->gpuNameStringFlags)
    {
        case NV2080_CTRL_GPU_GET_NAME_STRING_FLAGS_TYPE_ASCII:
            if (!cachedParams->bAsciiValid)
            {
                status = NV_ERR_OBJECT_NOT_FOUND;
                goto done;
            }
            portMemCopy(pParams->gpuNameString.ascii,
                        sizeof(pParams->gpuNameString.ascii),
                        cachedParams->ascii,
                        sizeof(pParams->gpuNameString.ascii));
            break;

        case NV2080_CTRL_GPU_GET_NAME_STRING_FLAGS_TYPE_UNICODE:
            if (!cachedParams->bUnicodeValid)
            {
                status = NV_ERR_OBJECT_NOT_FOUND;
                goto done;
            }
            portMemCopy(pParams->gpuNameString.unicode,
                        sizeof(pParams->gpuNameString.unicode),
                        cachedParams->unicode,
                        sizeof(pParams->gpuNameString.unicode));
            break;

        default:
            NV_PRINTF(LEVEL_ERROR, "Unknown gpu name string flag: %u\n", pParams->gpuNameStringFlags);
            status = NV_ERR_OBJECT_NOT_FOUND;
            goto done;
    }
done:
    _cacheLockRelease(LOCK_SHARED);
    return status;
}

NV_STATUS _gpuNameStringSet
(
    NvHandle hClient,
    NvHandle hObject,
    const NV2080_CTRL_GPU_GET_NAME_STRING_PARAMS *pParams
)
{
    NvU32 gpuInst;
    NV_STATUS status;
    RmapiControlCacheEntry *entry = NULL;
    GpuNameStringCacheEntry *cachedParams = NULL;

    _cacheLockAcquire(LOCK_EXCLUSIVE);

    if (rmapiControlCacheGetMode() == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_DISABLE)
    {
        // unexpected mode change.
        status = NV_ERR_INVALID_STATE;
        goto done;
    }

    status = _rmapiControlCacheGetGpuInstForObject(hClient, hObject, &gpuInst);
    if (status != NV_OK)
        goto done;

    entry = _getOrInitCacheEntry(gpuInst, NV2080_CTRL_CMD_GPU_GET_NAME_STRING,
                                 NV_TRUE, sizeof(GpuNameStringCacheEntry), NULL);
    if (entry == NULL)
    {
        status = NV_ERR_NO_MEMORY;
        goto done;
    }

    cachedParams = (GpuNameStringCacheEntry *)entry->params;

    switch (pParams->gpuNameStringFlags)
    {
        case NV2080_CTRL_GPU_GET_NAME_STRING_FLAGS_TYPE_ASCII:
            if (cachedParams->bAsciiValid)
            {
                NV_ASSERT(
                    portMemCmp(pParams->gpuNameString.ascii,
                               cachedParams->ascii,
                               sizeof(pParams->gpuNameString.ascii)) == 0);
            }
            else
            {
                portMemCopy(cachedParams->ascii,
                            sizeof(pParams->gpuNameString.ascii),
                            pParams->gpuNameString.ascii,
                            sizeof(pParams->gpuNameString.ascii));
                cachedParams->bAsciiValid = NV_TRUE;
            }
            break;

        case NV2080_CTRL_GPU_GET_NAME_STRING_FLAGS_TYPE_UNICODE:
            if (cachedParams->bUnicodeValid)
            {
                NV_ASSERT(
                    portMemCmp(pParams->gpuNameString.unicode,
                               cachedParams->unicode,
                               sizeof(pParams->gpuNameString.unicode)) == 0);
            }
            else
            {
                portMemCopy(cachedParams->unicode,
                            sizeof(pParams->gpuNameString.unicode),
                            pParams->gpuNameString.unicode,
                            sizeof(pParams->gpuNameString.unicode));
                cachedParams->bUnicodeValid = NV_TRUE;
            }
            break;

        default:
            NV_PRINTF(LEVEL_ERROR, "Unknown gpu name string flag: %u\n", pParams->gpuNameStringFlags);
            status = NV_ERR_INVALID_PARAMETER;
            goto done;
    }

done:
    if (status != NV_OK)
    {
        if (entry != NULL)
        {
            if (entry->params)
                portMemFree(entry->params);
            multimapRemoveItem(&RmapiControlCache.gpusControlCache, entry);
        }
    }
    _cacheLockRelease(LOCK_EXCLUSIVE);

    return status;
}

NV_STATUS _rmapiControlCacheGetByInput
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 cmd,
    void* params,
    NvU32 paramsSize
)
{
    switch (cmd)
    {
        case NV2080_CTRL_CMD_GPU_GET_INFO_V2:
            return _getInfoCacheHandler(hClient, hObject, cmd,
                                        ((NV2080_CTRL_GPU_GET_INFO_V2_PARAMS*)params)->gpuInfoList,
                                        ((NV2080_CTRL_GPU_GET_INFO_V2_PARAMS*)params)->gpuInfoListSize,
                                        NV2080_CTRL_GPU_INFO_MAX_LIST_SIZE,
                                        NV_FALSE);

        case NV2080_CTRL_CMD_FIFO_GET_INFO:
            return _getInfoCacheHandler(hClient, hObject, cmd,
                                        ((NV2080_CTRL_FIFO_GET_INFO_PARAMS*)params)->fifoInfoTbl,
                                        ((NV2080_CTRL_FIFO_GET_INFO_PARAMS*)params)->fifoInfoTblSize,
                                        NV2080_CTRL_FIFO_GET_INFO_MAX_ENTRIES,
                                        NV_FALSE);

        case NV2080_CTRL_CMD_BUS_GET_INFO_V2:
            return _getInfoCacheHandler(hClient, hObject, cmd,
                                        ((NV2080_CTRL_BUS_GET_INFO_V2_PARAMS*)params)->busInfoList,
                                        ((NV2080_CTRL_BUS_GET_INFO_V2_PARAMS*)params)->busInfoListSize,
                                        NV2080_CTRL_BUS_INFO_MAX_LIST_SIZE,
                                        NV_FALSE);

        case NV2080_CTRL_CMD_GPU_GET_NAME_STRING:
            return _gpuNameStringGet(hClient, hObject, params);
        default:
            NV_PRINTF(LEVEL_WARNING, "No implementation for cacheable by input cmd 0x%x\n", cmd);
            return NV_ERR_OBJECT_NOT_FOUND;
    }
}

NV_STATUS _rmapiControlCacheSetByInput
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 cmd,
    void* params,
    NvU32 paramsSize
)
{
    switch (cmd)
    {
        case NV2080_CTRL_CMD_GPU_GET_INFO_V2:
            return _getInfoCacheHandler(hClient, hObject, cmd,
                                        ((NV2080_CTRL_GPU_GET_INFO_V2_PARAMS*)params)->gpuInfoList,
                                        ((NV2080_CTRL_GPU_GET_INFO_V2_PARAMS*)params)->gpuInfoListSize,
                                        NV2080_CTRL_GPU_INFO_MAX_LIST_SIZE,
                                        NV_TRUE);

        case NV2080_CTRL_CMD_FIFO_GET_INFO:
            return _getInfoCacheHandler(hClient, hObject, cmd,
                                        ((NV2080_CTRL_FIFO_GET_INFO_PARAMS*)params)->fifoInfoTbl,
                                        ((NV2080_CTRL_FIFO_GET_INFO_PARAMS*)params)->fifoInfoTblSize,
                                        NV2080_CTRL_FIFO_GET_INFO_MAX_ENTRIES,
                                        NV_TRUE);

        case NV2080_CTRL_CMD_BUS_GET_INFO_V2:
            return _getInfoCacheHandler(hClient, hObject, cmd,
                                        ((NV2080_CTRL_BUS_GET_INFO_V2_PARAMS*)params)->busInfoList,
                                        ((NV2080_CTRL_BUS_GET_INFO_V2_PARAMS*)params)->busInfoListSize,
                                        NV2080_CTRL_BUS_INFO_MAX_LIST_SIZE,
                                        NV_TRUE);

        case NV2080_CTRL_CMD_GPU_GET_NAME_STRING:
            return _gpuNameStringSet(hClient, hObject, params);
        default:
            NV_PRINTF(LEVEL_WARNING, "No implementation for cacheable by input cmd 0x%x\n", cmd);
            return NV_ERR_OBJECT_NOT_FOUND;
    }
}

NV_STATUS rmapiControlCacheGet
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 cmd,
    void* params,
    NvU32 paramsSize
)
{
    NV_STATUS status = NV_OK;
    NvU32 flags = 0;

    if (RmapiControlCache.mode == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_VERIFY_ONLY)
        return NV_ERR_OBJECT_NOT_FOUND;

    status = rmapiutilGetControlInfo(cmd, &flags, NULL);
    if (status != NV_OK)
        goto done;

    switch ((flags & RMCTRL_FLAGS_CACHEABLE_ANY))
    {
        case RMCTRL_FLAGS_CACHEABLE:
            status = _rmapiControlCacheGet(hClient, hObject, cmd, params, paramsSize);
            break;
        case RMCTRL_FLAGS_CACHEABLE_BY_INPUT:
            status = _rmapiControlCacheGetByInput(hClient, hObject, cmd, params, paramsSize);
            break;
        default:
            NV_PRINTF(LEVEL_ERROR, "Invalid cacheable flag 0x%x for cmd 0x%x\n", flags, cmd);
            status = NV_ERR_INVALID_PARAMETER;
            goto done;
    }

done:
    NV_PRINTF(LEVEL_INFO, "control cache get for 0x%x 0x%x 0x%x status: 0x%x\n", hClient, hObject, cmd, status);
    return status;
}

NV_STATUS rmapiControlCacheSet
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 cmd,
    void* params,
    NvU32 paramsSize
)
{
    NV_STATUS status = NV_OK;
    NvU32 flags = 0;

    status = rmapiutilGetControlInfo(cmd, &flags, NULL);
    if (status != NV_OK)
        goto done;

    switch ((flags & RMCTRL_FLAGS_CACHEABLE_ANY))
    {
        case RMCTRL_FLAGS_CACHEABLE:
            status = _rmapiControlCacheSet(hClient, hObject, cmd, params, paramsSize);
            break;
        case RMCTRL_FLAGS_CACHEABLE_BY_INPUT:
            status = _rmapiControlCacheSetByInput(hClient, hObject, cmd, params, paramsSize);
            break;
        default:
            NV_PRINTF(LEVEL_ERROR, "Invalid cacheable flag 0x%x for cmd 0x%x\n", flags, cmd);
            status = NV_ERR_INVALID_PARAMETER;
            goto done;
    }

done:
    NV_PRINTF(LEVEL_INFO, "control cache set for 0x%x 0x%x 0x%x status: 0x%x\n", hClient, hObject, cmd, status);
    return status;
}

// Need to hold rmapi control cache write lock
static void _freeSubmap(GpusControlCacheSubmap* submap)
{
    /* (Sub)map modification invalidates the iterator, so we have to restart */
    while (NV_TRUE)
    {
        GpusControlCacheIter it = multimapSubmapIterItems(&RmapiControlCache.gpusControlCache, submap);

        if (multimapItemIterNext(&it))
        {
            RmapiControlCacheEntry* entry = it.pValue;
            portMemFree(entry->params);
            multimapRemoveItem(&RmapiControlCache.gpusControlCache, entry);
        }
        else
        {
            break;
        }
    }
    multimapRemoveSubmap(&RmapiControlCache.gpusControlCache, submap);
}

void rmapiControlCacheFreeAllCacheForGpu
(
    NvU32 gpuInst
)
{
    GpusControlCacheSubmap* submap;

    _cacheLockAcquire(LOCK_EXCLUSIVE);

    submap = multimapFindSubmap(&RmapiControlCache.gpusControlCache, gpuInst);

    if (submap != NULL)
        _freeSubmap(submap);

    _cacheLockRelease(LOCK_EXCLUSIVE);
}

void rmapiControlCacheFreeClientEntry(NvHandle hClient)
{
    _cacheLockAcquire(LOCK_EXCLUSIVE);
    _rmapiControlCacheFreeGpuInstForClient(hClient);
    _cacheLockRelease(LOCK_EXCLUSIVE);
}

void rmapiControlCacheFreeObjectEntry(NvHandle hClient, NvHandle hObject)
{
    if (hClient == hObject)
    {
        rmapiControlCacheFreeClientEntry(hClient);
        return;
    }

    _cacheLockAcquire(LOCK_EXCLUSIVE);
    _rmapiControlCacheFreeGpuInstForObject(hClient, hObject);
    _cacheLockRelease(LOCK_EXCLUSIVE);
}

void rmapiControlCacheFree(void)
{
    GpusControlCacheIter it;

    it = multimapItemIterAll(&RmapiControlCache.gpusControlCache);
    while (multimapItemIterNext(&it))
    {
        RmapiControlCacheEntry* entry = it.pValue;
        portMemFree(entry->params);
    }

    multimapDestroy(&RmapiControlCache.gpusControlCache);
    mapDestroy(&RmapiControlCache.objectToGpuInstMap);
    portSyncRwLockDestroy(RmapiControlCache.pLock);
}

void rmapiControlCacheSetMode(NvU32 mode)
{
    NV_PRINTF(LEVEL_INFO, "Set rmapi control cache mode to 0x%x\n", mode);

    _cacheLockAcquire(LOCK_EXCLUSIVE);
    RmapiControlCache.mode = mode;
    _cacheLockRelease(LOCK_EXCLUSIVE);
}

NvU32 rmapiControlCacheGetMode(void)
{
    return RmapiControlCache.mode;
}
