Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

MdeModulePkg: Add alternative queue sizes in NVME driver #6024

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

MdeModulePkg: Add alternative queue sizes in NVME driver #6024

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
113797e
MdeModulePkg: NvmExpressDxe: Correct function parameter modifer
makubacki Aug 9, 2023
ea8cd1b
MdeModulePkg: NvmExpressDxe: Improve NVMe controller init robustness
makubacki Aug 9, 2023
c419dc8
MdeModulePkg: Add alternative queue sizes in nvme driver
chris-oo Oct 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 27 additions & 6 deletions MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,8 @@ ProcessAsyncTaskList (
EFI_BLOCK_IO2_TOKEN *Token;
BOOLEAN HasNewItem;
EFI_STATUS Status;
UINT16 QueueSize = PcdGetBool (PcdSupportAlternativeQueueSize) ?
NVME_ALTERNATIVE_MAX_QUEUE_SIZE : NVME_ASYNC_CCQ_SIZE;

Private = (NVME_CONTROLLER_PRIVATE_DATA *)Context;
QueueId = 2;
Expand Down Expand Up @@ -724,7 +726,7 @@ ProcessAsyncTaskList (
}

Private->CqHdbl[QueueId].Cqh++;
if (Private->CqHdbl[QueueId].Cqh > MIN (NVME_ASYNC_CCQ_SIZE, Private->Cap.Mqes)) {
if (Private->CqHdbl[QueueId].Cqh > MIN (QueueSize, Private->Cap.Mqes)) {
Private->CqHdbl[QueueId].Cqh = 0;
Private->Pt[QueueId] ^= 1;
}
Expand Down Expand Up @@ -957,6 +959,8 @@ NvmExpressDriverBindingStart (
EFI_PHYSICAL_ADDRESS MappedAddr;
UINTN Bytes;
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru;
UINTN QueuePageCount = PcdGetBool (PcdSupportAlternativeQueueSize) ?
NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES : 6;

DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: start\n"));

Expand Down Expand Up @@ -1029,6 +1033,10 @@ NvmExpressDriverBindingStart (
}

//
// Depending on PCD disablement, either support the default or alternative
// queue sizes.
//
// Default:
// 6 x 4kB aligned buffers will be carved out of this buffer.
// 1st 4kB boundary is the start of the admin submission queue.
// 2nd 4kB boundary is the start of the admin completion queue.
Expand All @@ -1039,19 +1047,30 @@ NvmExpressDriverBindingStart (
//
// Allocate 6 pages of memory, then map it for bus master read and write.
//
// Alternative:
// 15 x 4kB aligned buffers will be carved out of this buffer.
// 1st 4kB boundary is the start of the admin submission queue.
// 5th 4kB boundary is the start of the admin completion queue.
// 6th 4kB boundary is the start of I/O submission queue #1.
// 10th 4kB boundary is the start of I/O completion queue #1.
// 11th 4kB boundary is the start of I/O submission queue #2.
// 15th 4kB boundary is the start of I/O completion queue #2.
//
// Allocate 15 pages of memory, then map it for bus master read and write.
//
Status = PciIo->AllocateBuffer (
PciIo,
AllocateAnyPages,
EfiBootServicesData,
6,
QueuePageCount,
(VOID **)&Private->Buffer,
0
);
if (EFI_ERROR (Status)) {
goto Exit;
}

Bytes = EFI_PAGES_TO_SIZE (6);
Bytes = EFI_PAGES_TO_SIZE (QueuePageCount);
Status = PciIo->Map (
PciIo,
EfiPciIoOperationBusMasterCommonBuffer,
Expand All @@ -1061,7 +1080,7 @@ NvmExpressDriverBindingStart (
&Private->Mapping
);

if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (QueuePageCount))) {
goto Exit;
}

Expand Down Expand Up @@ -1171,7 +1190,7 @@ NvmExpressDriverBindingStart (
}

if ((Private != NULL) && (Private->Buffer != NULL)) {
PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
PciIo->FreeBuffer (PciIo, QueuePageCount, Private->Buffer);
}

if ((Private != NULL) && (Private->ControllerData != NULL)) {
Expand Down Expand Up @@ -1247,6 +1266,8 @@ NvmExpressDriverBindingStop (
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *PassThru;
BOOLEAN IsEmpty;
EFI_TPL OldTpl;
UINT16 QueuePageCount = PcdGetBool (PcdSupportAlternativeQueueSize) ?
NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES : 6;

if (NumberOfChildren == 0) {
Status = gBS->OpenProtocol (
Expand Down Expand Up @@ -1293,7 +1314,7 @@ NvmExpressDriverBindingStop (
}

if (Private->Buffer != NULL) {
Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);
Private->PciIo->FreeBuffer (Private->PciIo, QueuePageCount, Private->Buffer);
}

FreePool (Private->ControllerData);
Expand Down
11 changes: 11 additions & 0 deletions MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,17 @@ extern EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiV

#define NVME_MAX_QUEUES 3 // Number of queues supported by the driver

//
// NVMe DXE to accommodate hardware which requires queue size 255.
// Driver supports queue size up to 255 (4 page SQ buffer).
// DXE driver creates queue size MIN(Cap.Mqes, NVME_MAX_QUEUE_SIZE) for all queues.
// Driver allocates queue buffer to support 255 max queue size.
// Each submission queue buffer is allocated as 64B * 256 = 4 * 4kB = 4 pages.
// Each completion queue buffer is allocated as 16B * 256 = 4kB = 1 page.
//
#define NVME_ALTERNATIVE_MAX_QUEUE_SIZE 255
#define NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES NVME_MAX_QUEUES * 5

//
// FormatNVM Admin Command LBA Format (LBAF) Mask
//
Expand Down
3 changes: 3 additions & 0 deletions MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@
gMediaSanitizeProtocolGuid ## PRODUCES
gEfiResetNotificationProtocolGuid ## CONSUMES

[Pcd]
gEfiMdeModulePkgTokenSpaceGuid.PcdSupportAlternativeQueueSize ## CONSUMES

# [Event]
# EVENT_TYPE_RELATIVE_TIMER ## SOMETIMES_CONSUMES
#
Expand Down
110 changes: 78 additions & 32 deletions MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ UINTN mNvmeControllerNumber = 0;
**/
EFI_STATUS
ReadNvmeControllerCapabilities (
IN NVME_CONTROLLER_PRIVATE_DATA *Private,
IN NVME_CAP *Cap
IN NVME_CONTROLLER_PRIVATE_DATA *Private,
OUT NVME_CAP *Cap
)
{
EFI_PCI_IO_PROTOCOL *PciIo;
Expand Down Expand Up @@ -604,14 +604,14 @@ NvmeCreateIoCompletionQueue (
CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
CommandPacket.QueueType = NVME_ADMIN_QUEUE;

if (Index == 1) {
if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes);
} else if (Index == 1) {
QueueSize = NVME_CCQ_SIZE;
} else if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) {
QueueSize = NVME_ASYNC_CCQ_SIZE;
} else {
if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) {
QueueSize = NVME_ASYNC_CCQ_SIZE;
} else {
QueueSize = Private->Cap.Mqes;
}
QueueSize = Private->Cap.Mqes;
}

CrIoCq.Qid = Index;
Expand Down Expand Up @@ -676,14 +676,14 @@ NvmeCreateIoSubmissionQueue (
CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
CommandPacket.QueueType = NVME_ADMIN_QUEUE;

if (Index == 1) {
QueueSize = NVME_CSQ_SIZE;
if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes);
} else if (Index == 1) {
QueueSize = NVME_CCQ_SIZE;
} else if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) {
QueueSize = NVME_ASYNC_CCQ_SIZE;
} else {
if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) {
QueueSize = NVME_ASYNC_CSQ_SIZE;
} else {
QueueSize = Private->Cap.Mqes;
}
QueueSize = Private->Cap.Mqes;
}

CrIoSq.Qid = Index;
Expand Down Expand Up @@ -730,13 +730,34 @@ NvmeControllerInit (
NVME_AQA Aqa;
NVME_ASQ Asq;
NVME_ACQ Acq;
UINT16 VidDid[2];
UINT8 Sn[21];
UINT8 Mn[41];

PciIo = Private->PciIo;

//
// Verify the controller is still accessible
//
Status = PciIo->Pci.Read (
VivianNK marked this conversation as resolved.
Show resolved Hide resolved
PciIo,
EfiPciIoWidthUint16,
PCI_VENDOR_ID_OFFSET,
ARRAY_SIZE (VidDid),
VidDid
);
if (EFI_ERROR (Status)) {
ASSERT_EFI_ERROR (Status);
return EFI_DEVICE_ERROR;
}

if ((VidDid[0] == 0xFFFF) || (VidDid[1] == 0xFFFF)) {
return EFI_DEVICE_ERROR;
}

//
// Enable this controller.
//
PciIo = Private->PciIo;
Status = PciIo->Attributes (
PciIo,
EfiPciIoAttributeOperationSupported,
Expand Down Expand Up @@ -774,8 +795,13 @@ NvmeControllerInit (

//
// Currently the driver only supports 4k page size.
// Currently, this means Cap.Mpsmin must be zero for an EFI_PAGE_SHIFT size of 12.
//
ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT);
if ((Private->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) {
DEBUG ((DEBUG_ERROR, "NvmeControllerInit: Mpsmin is larger than expected (0x%02x).\n", Private->Cap.Mpsmin));
return EFI_DEVICE_ERROR;
}

Private->Cid[0] = 0;
Private->Cid[1] = 0;
Expand All @@ -800,9 +826,9 @@ NvmeControllerInit (
//
// set number of entries admin submission & completion queues.
//
Aqa.Asqs = NVME_ASQ_SIZE;
Aqa.Asqs = PcdGetBool (PcdSupportAlternativeQueueSize) ? MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) : NVME_ASQ_SIZE;
Aqa.Rsvd1 = 0;
Aqa.Acqs = NVME_ACQ_SIZE;
Aqa.Acqs = PcdGetBool (PcdSupportAlternativeQueueSize) ? MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) : NVME_ACQ_SIZE;
Aqa.Rsvd2 = 0;

//
Expand All @@ -813,24 +839,44 @@ NvmeControllerInit (
//
// Address of admin completion queue.
//
Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF;
if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
Acq = (UINT64)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE) & ~0xFFF;
} else {
Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF;
}

//
// Address of I/O submission & completion queue.
//
ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6));
Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer);
Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);
Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE);
Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE);
Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);
Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);
Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE);
Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE);
Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE);
if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES));
Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer);
Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);
Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE);
Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE);
Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE);
Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 9 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 9 * EFI_PAGE_SIZE);
Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 10 * EFI_PAGE_SIZE);
Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 10 * EFI_PAGE_SIZE);
Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 14 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 14 * EFI_PAGE_SIZE);
} else {
ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6));
Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer);
Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);
Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE);
Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE);
Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);
Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);
Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE);
Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE);
Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE);
}

DEBUG ((DEBUG_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer));
DEBUG ((DEBUG_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
Expand Down
43 changes: 33 additions & 10 deletions MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,12 @@ NvmExpressPassThru (
Prp = NULL;
TimerEvent = NULL;
Status = EFI_SUCCESS;
QueueSize = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes) + 1;

if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) + 1;
} else {
QueueSize = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes) + 1;
}

if (Packet->QueueType == NVME_ADMIN_QUEUE) {
QueueId = 0;
Expand Down Expand Up @@ -581,6 +586,13 @@ NvmExpressPassThru (
return EFI_INVALID_PARAMETER;
}

//
// Nvme DXE driver polls phase bit for CQe completion.
// Explicitly assign phase bit with the bit before completion.
// A flipped phase bit will be assigned by device upon CQe completes.
//
Cq->Pt = Private->Pt[QueueId];

ZeroMem (Sq, sizeof (NVME_SQ));
Sq->Opc = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
Expand Down Expand Up @@ -723,14 +735,18 @@ NvmExpressPassThru (
Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
}

//
// Ring the submission queue doorbell.
//
if ((Event != NULL) && (QueueId != 0)) {
Private->SqTdbl[QueueId].Sqt =
(Private->SqTdbl[QueueId].Sqt + 1) % QueueSize;
if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
Private->SqTdbl[QueueId].Sqt = (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize;
} else {
Private->SqTdbl[QueueId].Sqt ^= 1;
//
// Ring the submission queue doorbell.
//
if ((Event != NULL) && (QueueId != 0)) {
Private->SqTdbl[QueueId].Sqt =
(Private->SqTdbl[QueueId].Sqt + 1) % QueueSize;
} else {
Private->SqTdbl[QueueId].Sqt ^= 1;
}
}

Data = ReadUnaligned32 ((UINT32 *)&Private->SqTdbl[QueueId]);
Expand Down Expand Up @@ -865,8 +881,15 @@ NvmExpressPassThru (
goto EXIT;
}

if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
Private->Pt[QueueId] ^= 1;
if (PcdGetBool (PcdSupportAlternativeQueueSize)) {
Private->CqHdbl[QueueId].Cqh = (Private->CqHdbl[QueueId].Cqh + 1) % QueueSize;
if (Private->CqHdbl[QueueId].Cqh == 0) {
Private->Pt[QueueId] ^= 1;
}
} else {
if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
Private->Pt[QueueId] ^= 1;
}
}

Data = ReadUnaligned32 ((UINT32 *)&Private->CqHdbl[QueueId]);
Expand Down
4 changes: 4 additions & 0 deletions MdeModulePkg/MdeModulePkg.dec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1198,6 +1198,10 @@
# @Prompt Defines the page allocation for the MM communication buffer; default is 128 pages (512KB).
gEfiMdeModulePkgTokenSpaceGuid.PcdMmCommBufferPages|128|UINT32|0x30001061

## Support for the alternative queue size.
# @Prompt TRUE to enable support for alternative queue size in NVME driver.
gEfiMdeModulePkgTokenSpaceGuid.PcdSupportAlternativeQueueSize|FALSE|BOOLEAN|0x40000151

[PcdsFixedAtBuild, PcdsPatchableInModule]
## Dynamic type PCD can be registered callback function for Pcd setting action.
# PcdMaxPeiPcdCallBackNumberPerPcdEntry indicates the maximum number of callback function
Expand Down
Loading