diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c index c8d8be32b0c8..0c344716486d 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c @@ -582,6 +582,8 @@ ProcessAsyncTaskList ( EFI_BLOCK_IO2_TOKEN *Token; BOOLEAN HasNewItem; EFI_STATUS Status; + UINT16 QueueSize = PcdGetBool (PcdSupportAlternativeQueueSize) ? + NVME_ALTERNATIVE_MAX_QUEUE_SIZE : NVME_ASYNC_CCQ_SIZE; Private = (NVME_CONTROLLER_PRIVATE_DATA *)Context; QueueId = 2; @@ -724,7 +726,7 @@ ProcessAsyncTaskList ( } Private->CqHdbl[QueueId].Cqh++; - if (Private->CqHdbl[QueueId].Cqh > MIN (NVME_ASYNC_CCQ_SIZE, Private->Cap.Mqes)) { + if (Private->CqHdbl[QueueId].Cqh > MIN (QueueSize, Private->Cap.Mqes)) { Private->CqHdbl[QueueId].Cqh = 0; Private->Pt[QueueId] ^= 1; } @@ -957,6 +959,8 @@ NvmExpressDriverBindingStart ( EFI_PHYSICAL_ADDRESS MappedAddr; UINTN Bytes; EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru; + UINTN QueuePageCount = PcdGetBool (PcdSupportAlternativeQueueSize) ? + NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES : 6; DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: start\n")); @@ -1029,6 +1033,10 @@ NvmExpressDriverBindingStart ( } // + // Depending on PCD disablement, either support the default or alternative + // queue sizes. + // + // Default: // 6 x 4kB aligned buffers will be carved out of this buffer. // 1st 4kB boundary is the start of the admin submission queue. // 2nd 4kB boundary is the start of the admin completion queue. @@ -1039,11 +1047,22 @@ NvmExpressDriverBindingStart ( // // Allocate 6 pages of memory, then map it for bus master read and write. // + // Alternative: + // 15 x 4kB aligned buffers will be carved out of this buffer. + // 1st 4kB boundary is the start of the admin submission queue. + // 5th 4kB boundary is the start of the admin completion queue. + // 6th 4kB boundary is the start of I/O submission queue #1. + // 10th 4kB boundary is the start of I/O completion queue #1. + // 11th 4kB boundary is the start of I/O submission queue #2. + // 15th 4kB boundary is the start of I/O completion queue #2. + // + // Allocate 15 pages of memory, then map it for bus master read and write. + // Status = PciIo->AllocateBuffer ( PciIo, AllocateAnyPages, EfiBootServicesData, - 6, + QueuePageCount, (VOID **)&Private->Buffer, 0 ); @@ -1051,7 +1070,7 @@ NvmExpressDriverBindingStart ( goto Exit; } - Bytes = EFI_PAGES_TO_SIZE (6); + Bytes = EFI_PAGES_TO_SIZE (QueuePageCount); Status = PciIo->Map ( PciIo, EfiPciIoOperationBusMasterCommonBuffer, @@ -1061,7 +1080,7 @@ NvmExpressDriverBindingStart ( &Private->Mapping ); - if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) { + if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (QueuePageCount))) { goto Exit; } @@ -1171,7 +1190,7 @@ NvmExpressDriverBindingStart ( } if ((Private != NULL) && (Private->Buffer != NULL)) { - PciIo->FreeBuffer (PciIo, 6, Private->Buffer); + PciIo->FreeBuffer (PciIo, QueuePageCount, Private->Buffer); } if ((Private != NULL) && (Private->ControllerData != NULL)) { @@ -1247,6 +1266,8 @@ NvmExpressDriverBindingStop ( EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *PassThru; BOOLEAN IsEmpty; EFI_TPL OldTpl; + UINT16 QueuePageCount = PcdGetBool (PcdSupportAlternativeQueueSize) ? + NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES : 6; if (NumberOfChildren == 0) { Status = gBS->OpenProtocol ( @@ -1293,7 +1314,7 @@ NvmExpressDriverBindingStop ( } if (Private->Buffer != NULL) { - Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer); + Private->PciIo->FreeBuffer (Private->PciIo, QueuePageCount, Private->Buffer); } FreePool (Private->ControllerData); diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h index 11207afa6332..30b47ff7909e 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h @@ -80,6 +80,17 @@ extern EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiV #define NVME_MAX_QUEUES 3 // Number of queues supported by the driver +// +// NVMe DXE to accommodate hardware which requires queue size 255. +// Driver supports queue size up to 255 (4 page SQ buffer). +// DXE driver creates queue size MIN(Cap.Mqes, NVME_MAX_QUEUE_SIZE) for all queues. +// Driver allocates queue buffer to support 255 max queue size. +// Each submission queue buffer is allocated as 64B * 256 = 4 * 4kB = 4 pages. +// Each completion queue buffer is allocated as 16B * 256 = 4kB = 1 page. +// +#define NVME_ALTERNATIVE_MAX_QUEUE_SIZE 255 +#define NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES NVME_MAX_QUEUES * 5 + // // FormatNVM Admin Command LBA Format (LBAF) Mask // diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf index 5fd0e468a922..5edd4faed5ee 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf @@ -77,6 +77,9 @@ gMediaSanitizeProtocolGuid ## PRODUCES gEfiResetNotificationProtocolGuid ## CONSUMES +[Pcd] + gEfiMdeModulePkgTokenSpaceGuid.PcdSupportAlternativeQueueSize ## CONSUMES + # [Event] # EVENT_TYPE_RELATIVE_TIMER ## SOMETIMES_CONSUMES # diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c index e1b0ee6051a6..071efa3bb033 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c @@ -30,8 +30,8 @@ UINTN mNvmeControllerNumber = 0; **/ EFI_STATUS ReadNvmeControllerCapabilities ( - IN NVME_CONTROLLER_PRIVATE_DATA *Private, - IN NVME_CAP *Cap + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + OUT NVME_CAP *Cap ) { EFI_PCI_IO_PROTOCOL *PciIo; @@ -604,14 +604,14 @@ NvmeCreateIoCompletionQueue ( CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; CommandPacket.QueueType = NVME_ADMIN_QUEUE; - if (Index == 1) { + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes); + } else if (Index == 1) { QueueSize = NVME_CCQ_SIZE; + } else if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) { + QueueSize = NVME_ASYNC_CCQ_SIZE; } else { - if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) { - QueueSize = NVME_ASYNC_CCQ_SIZE; - } else { - QueueSize = Private->Cap.Mqes; - } + QueueSize = Private->Cap.Mqes; } CrIoCq.Qid = Index; @@ -676,14 +676,14 @@ NvmeCreateIoSubmissionQueue ( CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; CommandPacket.QueueType = NVME_ADMIN_QUEUE; - if (Index == 1) { - QueueSize = NVME_CSQ_SIZE; + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes); + } else if (Index == 1) { + QueueSize = NVME_CCQ_SIZE; + } else if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) { + QueueSize = NVME_ASYNC_CCQ_SIZE; } else { - if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) { - QueueSize = NVME_ASYNC_CSQ_SIZE; - } else { - QueueSize = Private->Cap.Mqes; - } + QueueSize = Private->Cap.Mqes; } CrIoSq.Qid = Index; @@ -730,13 +730,34 @@ NvmeControllerInit ( NVME_AQA Aqa; NVME_ASQ Asq; NVME_ACQ Acq; + UINT16 VidDid[2]; UINT8 Sn[21]; UINT8 Mn[41]; + PciIo = Private->PciIo; + + // + // Verify the controller is still accessible + // + Status = PciIo->Pci.Read ( + PciIo, + EfiPciIoWidthUint16, + PCI_VENDOR_ID_OFFSET, + ARRAY_SIZE (VidDid), + VidDid + ); + if (EFI_ERROR (Status)) { + ASSERT_EFI_ERROR (Status); + return EFI_DEVICE_ERROR; + } + + if ((VidDid[0] == 0xFFFF) || (VidDid[1] == 0xFFFF)) { + return EFI_DEVICE_ERROR; + } + // // Enable this controller. // - PciIo = Private->PciIo; Status = PciIo->Attributes ( PciIo, EfiPciIoAttributeOperationSupported, @@ -774,8 +795,13 @@ NvmeControllerInit ( // // Currently the driver only supports 4k page size. + // Currently, this means Cap.Mpsmin must be zero for an EFI_PAGE_SHIFT size of 12. // ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT); + if ((Private->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) { + DEBUG ((DEBUG_ERROR, "NvmeControllerInit: Mpsmin is larger than expected (0x%02x).\n", Private->Cap.Mpsmin)); + return EFI_DEVICE_ERROR; + } Private->Cid[0] = 0; Private->Cid[1] = 0; @@ -800,9 +826,9 @@ NvmeControllerInit ( // // set number of entries admin submission & completion queues. // - Aqa.Asqs = NVME_ASQ_SIZE; + Aqa.Asqs = PcdGetBool (PcdSupportAlternativeQueueSize) ? MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) : NVME_ASQ_SIZE; Aqa.Rsvd1 = 0; - Aqa.Acqs = NVME_ACQ_SIZE; + Aqa.Acqs = PcdGetBool (PcdSupportAlternativeQueueSize) ? MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) : NVME_ACQ_SIZE; Aqa.Rsvd2 = 0; // @@ -813,24 +839,44 @@ NvmeControllerInit ( // // Address of admin completion queue. // - Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF; + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + Acq = (UINT64)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE) & ~0xFFF; + } else { + Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF; + } // // Address of I/O submission & completion queue. // - ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6)); - Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); - Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); - Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE); - Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE); - Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE); - Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE); - Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE); - Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE); - Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE); - Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE); - Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE); - Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE); + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES)); + Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); + Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); + Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE); + Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE); + Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 9 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 9 * EFI_PAGE_SIZE); + Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 10 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 10 * EFI_PAGE_SIZE); + Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 14 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 14 * EFI_PAGE_SIZE); + } else { + ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6)); + Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); + Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); + Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE); + Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE); + Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE); + Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE); + Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE); + } DEBUG ((DEBUG_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer)); DEBUG ((DEBUG_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs)); diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c index f818e48fc16d..c0472801cb5c 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c @@ -553,7 +553,12 @@ NvmExpressPassThru ( Prp = NULL; TimerEvent = NULL; Status = EFI_SUCCESS; - QueueSize = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes) + 1; + + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) + 1; + } else { + QueueSize = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes) + 1; + } if (Packet->QueueType == NVME_ADMIN_QUEUE) { QueueId = 0; @@ -581,6 +586,13 @@ NvmExpressPassThru ( return EFI_INVALID_PARAMETER; } + // + // Nvme DXE driver polls phase bit for CQe completion. + // Explicitly assign phase bit with the bit before completion. + // A flipped phase bit will be assigned by device upon CQe completes. + // + Cq->Pt = Private->Pt[QueueId]; + ZeroMem (Sq, sizeof (NVME_SQ)); Sq->Opc = (UINT8)Packet->NvmeCmd->Cdw0.Opcode; Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation; @@ -723,14 +735,18 @@ NvmExpressPassThru ( Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15; } - // - // Ring the submission queue doorbell. - // - if ((Event != NULL) && (QueueId != 0)) { - Private->SqTdbl[QueueId].Sqt = - (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize; + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + Private->SqTdbl[QueueId].Sqt = (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize; } else { - Private->SqTdbl[QueueId].Sqt ^= 1; + // + // Ring the submission queue doorbell. + // + if ((Event != NULL) && (QueueId != 0)) { + Private->SqTdbl[QueueId].Sqt = + (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize; + } else { + Private->SqTdbl[QueueId].Sqt ^= 1; + } } Data = ReadUnaligned32 ((UINT32 *)&Private->SqTdbl[QueueId]); @@ -865,8 +881,15 @@ NvmExpressPassThru ( goto EXIT; } - if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) { - Private->Pt[QueueId] ^= 1; + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + Private->CqHdbl[QueueId].Cqh = (Private->CqHdbl[QueueId].Cqh + 1) % QueueSize; + if (Private->CqHdbl[QueueId].Cqh == 0) { + Private->Pt[QueueId] ^= 1; + } + } else { + if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) { + Private->Pt[QueueId] ^= 1; + } } Data = ReadUnaligned32 ((UINT32 *)&Private->CqHdbl[QueueId]); diff --git a/MdeModulePkg/MdeModulePkg.dec b/MdeModulePkg/MdeModulePkg.dec index 1324b6d1000c..b4822c5f7b85 100644 --- a/MdeModulePkg/MdeModulePkg.dec +++ b/MdeModulePkg/MdeModulePkg.dec @@ -1198,6 +1198,10 @@ # @Prompt Defines the page allocation for the MM communication buffer; default is 128 pages (512KB). gEfiMdeModulePkgTokenSpaceGuid.PcdMmCommBufferPages|128|UINT32|0x30001061 + ## Support for the alternative queue size. + # @Prompt TRUE to enable support for alternative queue size in NVME driver. + gEfiMdeModulePkgTokenSpaceGuid.PcdSupportAlternativeQueueSize|FALSE|BOOLEAN|0x40000151 + [PcdsFixedAtBuild, PcdsPatchableInModule] ## Dynamic type PCD can be registered callback function for Pcd setting action. # PcdMaxPeiPcdCallBackNumberPerPcdEntry indicates the maximum number of callback function