[DM/FEATURE] Support NVME (#9591)

* [DM/FEATURE] Support NVME

1. Support PRP and SGL (>= NVME v1.1) transport.
2. Support MSI/MSI-X for IO queues.
3. Support NVME on PCI.

Signed-off-by: GuEe-GUI <[email protected]>

2 files changed, 903 insertions, 0 deletions

diff --git a/components/drivers/include/drivers/nvme.h b/components/drivers/include/drivers/nvme.h
new file mode 100644
index 0000000000..1eca5d9caa
--- /dev/null
+++ b/components/drivers/include/drivers/nvme.h
@@ -0,0 +1,899 @@
+/*
+ * Copyright (c) 2006-2023, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date           Author       Notes
+ * 2023-02-25     GuEe-GUI     the first version
+ */
+
+#ifndef __NVME_H__
+#define __NVME_H__
+
+#include <rthw.h>
+#include <rtthread.h>
+#include <drivers/blk.h>
+
+#define NVME_RSVD(offset, bytes_size)   rt_uint8_t __rsvd##offset[bytes_size]
+
+enum
+{
+    /*
+     * Generic Command Status:
+     */
+    RT_NVME_SC_SUCCESS                          = 0x0,
+    RT_NVME_SC_INVALID_OPCODE                   = 0x1,
+    RT_NVME_SC_INVALID_FIELD                    = 0x2,
+    RT_NVME_SC_CMDID_CONFLICT                   = 0x3,
+    RT_NVME_SC_DATA_XFER_ERROR                  = 0x4,
+    RT_NVME_SC_POWER_LOSS                       = 0x5,
+    RT_NVME_SC_INTERNAL                         = 0x6,
+    RT_NVME_SC_ABORT_REQ                        = 0x7,
+    RT_NVME_SC_ABORT_QUEUE                      = 0x8,
+    RT_NVME_SC_FUSED_FAIL                       = 0x9,
+    RT_NVME_SC_FUSED_MISSING                    = 0xa,
+    RT_NVME_SC_INVALID_NS                       = 0xb,
+    RT_NVME_SC_CMD_SEQ_ERROR                    = 0xc,
+    RT_NVME_SC_SGL_INVALID_LAST                 = 0xd,
+    RT_NVME_SC_SGL_INVALID_COUNT                = 0xe,
+    RT_NVME_SC_SGL_INVALID_DATA                 = 0xf,
+    RT_NVME_SC_SGL_INVALID_METADATA             = 0x10,
+    RT_NVME_SC_SGL_INVALID_TYPE                 = 0x11,
+    RT_NVME_SC_CMB_INVALID_USE                  = 0x12,
+    RT_NVME_SC_PRP_INVALID_OFFSET               = 0x13,
+    RT_NVME_SC_ATOMIC_WU_EXCEEDED               = 0x14,
+    RT_NVME_SC_OP_DENIED                        = 0x15,
+    RT_NVME_SC_SGL_INVALID_OFFSET               = 0x16,
+    RT_NVME_SC_RESERVED                         = 0x17,
+    RT_NVME_SC_HOST_ID_INCONSIST                = 0x18,
+    RT_NVME_SC_KA_TIMEOUT_EXPIRED               = 0x19,
+    RT_NVME_SC_KA_TIMEOUT_INVALID               = 0x1a,
+    RT_NVME_SC_ABORTED_PREEMPT_ABORT            = 0x1b,
+    RT_NVME_SC_SANITIZE_FAILED                  = 0x1c,
+    RT_NVME_SC_SANITIZE_IN_PROGRESS             = 0x1d,
+    RT_NVME_SC_SGL_INVALID_GRANULARITY          = 0x1e,
+    RT_NVME_SC_CMD_NOT_SUP_CMB_QUEUE            = 0x1f,
+    RT_NVME_SC_NS_WRITE_PROTECTED               = 0x20,
+    RT_NVME_SC_CMD_INTERRUPTED                  = 0x21,
+    RT_NVME_SC_TRANSIENT_TR_ERR                 = 0x22,
+    RT_NVME_SC_ADMIN_COMMAND_MEDIA_NOT_READY    = 0x24,
+    RT_NVME_SC_INVALID_IO_CMD_SET               = 0x2c,
+
+    RT_NVME_SC_LBA_RANGE                        = 0x80,
+    RT_NVME_SC_CAP_EXCEEDED                     = 0x81,
+    RT_NVME_SC_NS_NOT_READY                     = 0x82,
+    RT_NVME_SC_RESERVATION_CONFLICT             = 0x83,
+    RT_NVME_SC_FORMAT_IN_PROGRESS               = 0x84,
+
+    /*
+     * Command Specific Status:
+     */
+    RT_NVME_SC_CQ_INVALID                       = 0x100,
+    RT_NVME_SC_QID_INVALID                      = 0x101,
+    RT_NVME_SC_QUEUE_SIZE                       = 0x102,
+    RT_NVME_SC_ABORT_LIMIT                      = 0x103,
+    RT_NVME_SC_ABORT_MISSING                    = 0x104,
+    RT_NVME_SC_ASYNC_LIMIT                      = 0x105,
+    RT_NVME_SC_FIRMWARE_SLOT                    = 0x106,
+    RT_NVME_SC_FIRMWARE_IMAGE                   = 0x107,
+    RT_NVME_SC_INVALID_VECTOR                   = 0x108,
+    RT_NVME_SC_INVALID_LOG_PAGE                 = 0x109,
+    RT_NVME_SC_INVALID_FORMAT                   = 0x10a,
+    RT_NVME_SC_FW_NEEDS_CONV_RESET              = 0x10b,
+    RT_NVME_SC_INVALID_QUEUE                    = 0x10c,
+    RT_NVME_SC_FEATURE_NOT_SAVEABLE             = 0x10d,
+    RT_NVME_SC_FEATURE_NOT_CHANGEABLE           = 0x10e,
+    RT_NVME_SC_FEATURE_NOT_PER_NS               = 0x10f,
+    RT_NVME_SC_FW_NEEDS_SUBSYS_RESET            = 0x110,
+    RT_NVME_SC_FW_NEEDS_RESET                   = 0x111,
+    RT_NVME_SC_FW_NEEDS_MAX_TIME                = 0x112,
+    RT_NVME_SC_FW_ACTIVATE_PROHIBITED           = 0x113,
+    RT_NVME_SC_OVERLAPPING_RANGE                = 0x114,
+    RT_NVME_SC_NS_INSUFFICIENT_CAP              = 0x115,
+    RT_NVME_SC_NS_ID_UNAVAILABLE                = 0x116,
+    RT_NVME_SC_NS_ALREADY_ATTACHED              = 0x118,
+    RT_NVME_SC_NS_IS_PRIVATE                    = 0x119,
+    RT_NVME_SC_NS_NOT_ATTACHED                  = 0x11a,
+    RT_NVME_SC_THIN_PROV_NOT_SUPP               = 0x11b,
+    RT_NVME_SC_CTRL_LIST_INVALID                = 0x11c,
+    RT_NVME_SC_SELT_TEST_IN_PROGRESS            = 0x11d,
+    RT_NVME_SC_BP_WRITE_PROHIBITED              = 0x11e,
+    RT_NVME_SC_CTRL_ID_INVALID                  = 0x11f,
+    RT_NVME_SC_SEC_CTRL_STATE_INVALID           = 0x120,
+    RT_NVME_SC_CTRL_RES_NUM_INVALID             = 0x121,
+    RT_NVME_SC_RES_ID_INVALID                   = 0x122,
+    RT_NVME_SC_PMR_SAN_PROHIBITED               = 0x123,
+    RT_NVME_SC_ANA_GROUP_ID_INVALID             = 0x124,
+    RT_NVME_SC_ANA_ATTACH_FAILED                = 0x125,
+
+    /*
+     * I/O Command Set Specific - NVM commands:
+     */
+    RT_NVME_SC_BAD_ATTRIBUTES                   = 0x180,
+    RT_NVME_SC_INVALID_PI                       = 0x181,
+    RT_NVME_SC_READ_ONLY                        = 0x182,
+    RT_NVME_SC_ONCS_NOT_SUPPORTED               = 0x183,
+
+    /*
+     * I/O Command Set Specific - Fabrics commands:
+     */
+    RT_NVME_SC_CONNECT_FORMAT                   = 0x180,
+    RT_NVME_SC_CONNECT_CTRL_BUSY                = 0x181,
+    RT_NVME_SC_CONNECT_INVALID_PARAM            = 0x182,
+    RT_NVME_SC_CONNECT_RESTART_DISC             = 0x183,
+    RT_NVME_SC_CONNECT_INVALID_HOST             = 0x184,
+
+    RT_NVME_SC_DISCOVERY_RESTART                = 0x190,
+    RT_NVME_SC_AUTH_REQUIRED                    = 0x191,
+
+    /*
+     * I/O Command Set Specific - Zoned commands:
+     */
+    RT_NVME_SC_ZONE_BOUNDARY_ERROR              = 0x1b8,
+    RT_NVME_SC_ZONE_FULL                        = 0x1b9,
+    RT_NVME_SC_ZONE_READ_ONLY                   = 0x1ba,
+    RT_NVME_SC_ZONE_OFFLINE                     = 0x1bb,
+    RT_NVME_SC_ZONE_INVALID_WRITE               = 0x1bc,
+    RT_NVME_SC_ZONE_TOO_MANY_ACTIVE             = 0x1bd,
+    RT_NVME_SC_ZONE_TOO_MANY_OPEN               = 0x1be,
+    RT_NVME_SC_ZONE_INVALID_TRANSITION          = 0x1bf,
+
+    /*
+     * Media and Data Integrity Errors:
+     */
+    RT_NVME_SC_WRITE_FAULT                      = 0x280,
+    RT_NVME_SC_READ_ERROR                       = 0x281,
+    RT_NVME_SC_GUARD_CHECK                      = 0x282,
+    RT_NVME_SC_APPTAG_CHECK                     = 0x283,
+    RT_NVME_SC_REFTAG_CHECK                     = 0x284,
+    RT_NVME_SC_COMPARE_FAILED                   = 0x285,
+    RT_NVME_SC_ACCESS_DENIED                    = 0x286,
+    RT_NVME_SC_UNWRITTEN_BLOCK                  = 0x287,
+
+    /*
+     * Path-related Errors:
+     */
+    RT_NVME_SC_INTERNAL_PATH_ERROR              = 0x300,
+    RT_NVME_SC_ANA_PERSISTENT_LOSS              = 0x301,
+    RT_NVME_SC_ANA_INACCESSIBLE                 = 0x302,
+    RT_NVME_SC_ANA_TRANSITION                   = 0x303,
+    RT_NVME_SC_CTRL_PATH_ERROR                  = 0x360,
+    RT_NVME_SC_HOST_PATH_ERROR                  = 0x370,
+    RT_NVME_SC_HOST_ABORTED_CMD                 = 0x371,
+
+    RT_NVME_SC_CRD                              = 0x1800,
+    RT_NVME_SC_MORE                             = 0x2000,
+    RT_NVME_SC_DNR                              = 0x4000,
+};
+
+/* Admin commands */
+enum
+{
+    RT_NVME_ADMIN_OPCODE_DELETE_SQ          = 0x00,
+    RT_NVME_ADMIN_OPCODE_CREATE_SQ          = 0x01,
+    RT_NVME_ADMIN_OPCODE_GET_LOG_PAGE       = 0x02,
+    RT_NVME_ADMIN_OPCODE_DELETE_CQ          = 0x04,
+    RT_NVME_ADMIN_OPCODE_CREATE_CQ          = 0x05,
+    RT_NVME_ADMIN_OPCODE_IDENTIFY           = 0x06,
+    RT_NVME_ADMIN_OPCODE_ABORT_CMD          = 0x08,
+    RT_NVME_ADMIN_OPCODE_SET_FEATURES       = 0x09,
+    RT_NVME_ADMIN_OPCODE_GET_FEATURES       = 0x0a,
+    RT_NVME_ADMIN_OPCODE_ASYNC_EVENT        = 0x0c,
+    RT_NVME_ADMIN_OPCODE_NS_MGMT            = 0x0d,
+    RT_NVME_ADMIN_OPCODE_ACTIVATE_FW        = 0x10,
+    RT_NVME_ADMIN_OPCODE_DOWNLOAD_FW        = 0x11,
+    RT_NVME_ADMIN_OPCODE_DEV_SELF_TEST      = 0x14,
+    RT_NVME_ADMIN_OPCODE_NS_ATTACH          = 0x15,
+    RT_NVME_ADMIN_OPCODE_KEEP_ALIVE         = 0x18,
+    RT_NVME_ADMIN_OPCODE_DIRECTIVE_SEND     = 0x19,
+    RT_NVME_ADMIN_OPCODE_DIRECTIVE_RECV     = 0x1a,
+    RT_NVME_ADMIN_OPCODE_VIRTUAL_MGMT       = 0x1c,
+    RT_NVME_ADMIN_OPCODE_NVME_MI_SEND       = 0x1d,
+    RT_NVME_ADMIN_OPCODE_NVME_MI_RECV       = 0x1e,
+    RT_NVME_ADMIN_OPCODE_DBBUF              = 0x7c,
+    RT_NVME_ADMIN_OPCODE_FORMAT_NVM         = 0x80,
+    RT_NVME_ADMIN_OPCODE_SECURITY_SEND      = 0x81,
+    RT_NVME_ADMIN_OPCODE_SECURITY_RECV      = 0x82,
+    RT_NVME_ADMIN_OPCODE_SANITIZE_NVM       = 0x84,
+    RT_NVME_ADMIN_OPCODE_GET_LBA_STATUS     = 0x86,
+    RT_NVME_ADMIN_OPCODE_VENDOR_START       = 0xc0,
+};
+
+/* I/O commands */
+enum
+{
+    RT_NVME_CMD_FLUSH           = 0x00,
+    RT_NVME_CMD_WRITE           = 0x01,
+    RT_NVME_CMD_READ            = 0x02,
+    RT_NVME_CMD_WRITE_UNCOR     = 0x04,
+    RT_NVME_CMD_COMPARE         = 0x05,
+    RT_NVME_CMD_WRITE_ZEROES    = 0x08,
+    RT_NVME_CMD_DSM             = 0x09,
+    RT_NVME_CMD_VERIFY          = 0x0c,
+    RT_NVME_CMD_RESV_REGISTER   = 0x0d,
+    RT_NVME_CMD_RESV_REPORT     = 0x0e,
+    RT_NVME_CMD_RESV_ACQUIRE    = 0x11,
+    RT_NVME_CMD_RESV_RELEASE    = 0x15,
+    RT_NVME_CMD_ZONE_MGMT_SEND  = 0x79,
+    RT_NVME_CMD_ZONE_MGMT_RECV  = 0x7a,
+    RT_NVME_CMD_ZONE_APPEND     = 0x7d,
+    RT_NVME_CMD_VENDOR_START    = 0x80,
+};
+
+enum
+{
+    RT_NVME_PSDT_PRP                    = 0x0,
+    RT_NVME_PSDT_SGL_MPTR_CONTIGUOUS    = 0x1,
+    RT_NVME_PSDT_SGL_MPTR_SGL           = 0x2,
+};
+
+/* Commands flags */
+enum
+{
+    RT_NVME_CMD_FLAGS_FUSE_SHIFT        = 0x00,
+    RT_NVME_CMD_FLAGS_PSDT_SHIFT        = 0x06,
+};
+
+struct rt_nvme_command_common
+{
+    rt_uint8_t  opcode;
+    rt_uint8_t  flags;
+    rt_uint16_t cmdid;
+    rt_le32_t   nsid;
+    rt_le32_t   cmd_dw2[2];
+    rt_le64_t   metadata;
+    rt_le64_t   prp1;
+    rt_le64_t   prp2;
+    rt_le32_t   cmd_dw10[6];
+};
+
+rt_packed(struct rt_nvme_sgl_desc
+{
+    rt_le64_t adddress;
+    rt_le32_t length;
+    rt_uint8_t reserved[3];
+#define SGL_DESC_TYPE_DATA_BLOCK        0x0
+#define SGL_DESC_TYPE_BIT_BUCKET        0x1
+#define SGL_DESC_TYPE_SEGMENT           0x2
+#define SGL_DESC_TYPE_LAST_SEGMENT      0x3
+#define SGL_DESC_TYPE_KEYED_DATA_BLOCK  0x4
+#define SGL_DESC_TYPE_VENDOR_SPECIFIC   0xf
+    rt_uint8_t sgl_identify;
+});
+
+struct rt_nvme_command_rw
+{
+    rt_uint8_t  opcode;
+    rt_uint8_t  flags;
+    rt_uint16_t cmdid;
+    rt_le32_t   nsid;
+    NVME_RSVD(8, 8);
+    rt_le64_t   metadata;
+    union
+    {
+        struct
+        {
+            rt_le64_t prp1;
+            rt_le64_t prp2;
+        };
+        struct rt_nvme_sgl_desc sgl;
+    };
+    rt_le64_t   slba;
+    rt_le16_t   length;
+    rt_le16_t   control;
+    rt_le32_t   dsmgmt;
+    rt_le32_t   reftag;
+    rt_le16_t   apptag;
+    rt_le16_t   appmask;
+};
+
+enum
+{
+    RT_NVME_RW_LR                   = 1 << 15,
+    RT_NVME_RW_FUA                  = 1 << 14,
+    RT_NVME_RW_APPEND_PIREMAP       = 1 << 9,
+    RT_NVME_RW_DSM_FREQ_UNSPEC      = 0,
+    RT_NVME_RW_DSM_FREQ_TYPICAL     = 1,
+    RT_NVME_RW_DSM_FREQ_RARE        = 2,
+    RT_NVME_RW_DSM_FREQ_READS       = 3,
+    RT_NVME_RW_DSM_FREQ_WRITES      = 4,
+    RT_NVME_RW_DSM_FREQ_RW          = 5,
+    RT_NVME_RW_DSM_FREQ_ONCE        = 6,
+    RT_NVME_RW_DSM_FREQ_PREFETCH    = 7,
+    RT_NVME_RW_DSM_FREQ_TEMP        = 8,
+    RT_NVME_RW_DSM_LATENCY_NONE     = 0 << 4,
+    RT_NVME_RW_DSM_LATENCY_IDLE     = 1 << 4,
+    RT_NVME_RW_DSM_LATENCY_NORM     = 2 << 4,
+    RT_NVME_RW_DSM_LATENCY_LOW      = 3 << 4,
+    RT_NVME_RW_DSM_SEQ_REQ          = 1 << 6,
+    RT_NVME_RW_DSM_COMPRESSED       = 1 << 7,
+    RT_NVME_RW_PRINFO_PRCHK_REF     = 1 << 10,
+    RT_NVME_RW_PRINFO_PRCHK_APP     = 1 << 11,
+    RT_NVME_RW_PRINFO_PRCHK_GUARD   = 1 << 12,
+    RT_NVME_RW_PRINFO_PRACT         = 1 << 13,
+    RT_NVME_RW_DTYPE_STREAMS        = 1 << 4,
+    RT_NVME_WZ_DEAC                 = 1 << 9,
+};
+
+enum
+{
+    RT_NVME_QUEUE_PHYS_CONTIG   = (1 << 0),
+    RT_NVME_CQ_IRQ_ENABLED      = (1 << 1),
+    RT_NVME_SQ_PRIO_URGENT      = (0 << 1),
+    RT_NVME_SQ_PRIO_HIGH        = (1 << 1),
+    RT_NVME_SQ_PRIO_MEDIUM      = (2 << 1),
+    RT_NVME_SQ_PRIO_LOW         = (3 << 1),
+    RT_NVME_FEAT_ARBITRATION    = 0x01,
+    RT_NVME_FEAT_POWER_MGMT     = 0x02,
+    RT_NVME_FEAT_LBA_RANGE      = 0x03,
+    RT_NVME_FEAT_TEMP_THRESH    = 0x04,
+    RT_NVME_FEAT_ERR_RECOVERY   = 0x05,
+    RT_NVME_FEAT_VOLATILE_WC    = 0x06,
+    RT_NVME_FEAT_NUM_QUEUES     = 0x07,
+    RT_NVME_FEAT_IRQ_COALESCE   = 0x08,
+    RT_NVME_FEAT_IRQ_CONFIG     = 0x09,
+    RT_NVME_FEAT_WRITE_ATOMIC   = 0x0a,
+    RT_NVME_FEAT_ASYNC_EVENT    = 0x0b,
+    RT_NVME_FEAT_AUTO_PST       = 0x0c,
+    RT_NVME_FEAT_SW_PROGRESS    = 0x80,
+    RT_NVME_FEAT_HOST_ID        = 0x81,
+    RT_NVME_FEAT_RESV_MASK      = 0x82,
+    RT_NVME_FEAT_RESV_PERSIST   = 0x83,
+    RT_NVME_LOG_ERROR           = 0x01,
+    RT_NVME_LOG_SMART           = 0x02,
+    RT_NVME_LOG_FW_SLOT         = 0x03,
+    RT_NVME_LOG_RESERVATION     = 0x80,
+    RT_NVME_FWACT_REPL          = (0 << 3),
+    RT_NVME_FWACT_REPL_ACTV     = (1 << 3),
+    RT_NVME_FWACT_ACTV          = (2 << 3),
+};
+
+struct rt_nvme_command_identify
+{
+    rt_uint8_t  opcode;
+    rt_uint8_t  flags;
+    rt_uint16_t cmdid;
+    rt_le32_t   nsid;
+    NVME_RSVD(8, 16);
+    rt_le64_t   prp1;
+    rt_le64_t   prp2;
+    rt_le32_t   cns;
+    NVME_RSVD(64, 20);
+};
+
+struct rt_nvme_command_features
+{
+    rt_uint8_t  opcode;
+    rt_uint8_t  flags;
+    rt_uint16_t cmdid;
+    rt_le32_t   nsid;
+    NVME_RSVD(8, 16);
+    rt_le64_t   prp1;
+    rt_le64_t   prp2;
+    rt_le32_t   fid;
+    rt_le32_t   dword11;
+    NVME_RSVD(68, 16);
+};
+
+struct rt_nvme_command_create_cq
+{
+    rt_uint8_t  opcode;
+    rt_uint8_t  flags;
+    rt_uint16_t cmdid;
+    NVME_RSVD(4, 20);
+    rt_le64_t   prp1;
+    NVME_RSVD(32, 8);
+    rt_le16_t   cqid;
+    rt_le16_t   qsize;
+    rt_le16_t   cq_flags;
+    rt_le16_t   irq_vector;
+    NVME_RSVD(104, 16);
+};
+
+struct rt_nvme_command_create_sq
+{
+    rt_uint8_t  opcode;
+    rt_uint8_t  flags;
+    rt_uint16_t cmdid;
+    NVME_RSVD(4, 20);
+    rt_le64_t   prp1;
+    NVME_RSVD(32, 8);
+    rt_le16_t   sqid;
+    rt_le16_t   qsize;
+    rt_le16_t   sq_flags;
+    rt_le16_t   cqid;
+    NVME_RSVD(104, 16);
+};
+
+struct rt_nvme_command_delete_queue
+{
+    rt_uint8_t  opcode;
+    rt_uint8_t  flags;
+    rt_uint16_t cmdid;
+    NVME_RSVD(4, 36);
+    rt_le16_t   qid;
+    NVME_RSVD(42, 22);
+};
+
+struct rt_nvme_command_write_zeroes
+{
+    rt_uint8_t  opcode;
+    rt_uint8_t  flags;
+    rt_uint16_t cmdid;
+    rt_le32_t   nsid;
+    NVME_RSVD(8, 8);
+    rt_le64_t   metadata;
+    rt_le64_t   prp1;
+    rt_le64_t   prp2;
+    rt_le64_t   slba;
+    rt_le16_t   length;
+    rt_le16_t   control;
+    rt_le32_t   dsmgmt;
+    rt_le32_t   reftag;
+    rt_le16_t   apptag;
+    rt_le16_t   appmask;
+};
+
+struct rt_nvme_command
+{
+    union
+    {
+        struct rt_nvme_command_common common;
+        struct rt_nvme_command_rw rw;
+        struct rt_nvme_command_identify identify;
+        struct rt_nvme_command_features features;
+        struct rt_nvme_command_create_cq create_cq;
+        struct rt_nvme_command_create_sq create_sq;
+        struct rt_nvme_command_delete_queue delete_queue;
+        struct rt_nvme_command_write_zeroes write_zeroes;
+    };
+};
+
+struct rt_nvme_completion
+{
+    union
+    {
+        rt_le16_t  u16;
+        rt_le32_t  u32;
+        rt_le64_t  u64;
+    } result;
+    rt_le16_t   sq_head;    /* How much of this queue may be reclaimed */
+    rt_le16_t   sq_id;      /* Submission queue that generated this entry */
+    rt_uint16_t cmdid;      /* Which command completed */
+    rt_le16_t   status;     /* Command status */
+};
+
+enum
+{
+    RT_NVME_REG_CAP         = 0x0000,   /* Controller Capabilities */
+    RT_NVME_REG_VS          = 0x0008,   /* Version */
+    RT_NVME_REG_INTMS       = 0x000c,   /* Interrupt Mask Set */
+    RT_NVME_REG_INTMC       = 0x0010,   /* Interrupt Mask Clear */
+    RT_NVME_REG_CC          = 0x0014,   /* Controller Configuration */
+    RT_NVME_REG_CSTS        = 0x001c,   /* Controller Status */
+    RT_NVME_REG_NSSR        = 0x0020,   /* NVM Subsystem Reset */
+    RT_NVME_REG_AQA         = 0x0024,   /* Admin Queue Attributes */
+    RT_NVME_REG_ASQ         = 0x0028,   /* Admin SQ Base Address */
+    RT_NVME_REG_ACQ         = 0x0030,   /* Admin CQ Base Address */
+    RT_NVME_REG_CMBLOC      = 0x0038,   /* Controller Memory Buffer Location */
+    RT_NVME_REG_CMBSZ       = 0x003c,   /* Controller Memory Buffer Size */
+    RT_NVME_REG_BPINFO      = 0x0040,   /* Boot Partition Information */
+    RT_NVME_REG_BPRSEL      = 0x0044,   /* Boot Partition Read Select */
+    RT_NVME_REG_BPMBL       = 0x0048,   /* Boot Partition Memory Buffer Location */
+    RT_NVME_REG_CMBMSC      = 0x0050,   /* Controller Memory Buffer Memory Space Control */
+    RT_NVME_REG_CRTO        = 0x0068,   /* Controller Ready Timeouts */
+    RT_NVME_REG_PMRCAP      = 0x0e00,   /* Persistent Memory Capabilities */
+    RT_NVME_REG_PMRCTL      = 0x0e04,   /* Persistent Memory Region Control */
+    RT_NVME_REG_PMRSTS      = 0x0e08,   /* Persistent Memory Region Status */
+    RT_NVME_REG_PMREBS      = 0x0e0c,   /* Persistent Memory Region Elasticity Buffer Size */
+    RT_NVME_REG_PMRSWTP     = 0x0e10,   /* Persistent Memory Region Sustained Write Throughput */
+    RT_NVME_REG_DBS         = 0x1000,   /* SQ 0 Tail Doorbell */
+};
+
+#define RT_NVME_CAP_MQES(cap)       ((cap) & 0xffff)
+#define RT_NVME_CAP_TIMEOUT(cap)    (((cap) >> 24) & 0xff)
+#define RT_NVME_CAP_STRIDE(cap)     (((cap) >> 32) & 0xf)
+#define RT_NVME_CAP_MPSMIN(cap)     (((cap) >> 48) & 0xf)
+#define RT_NVME_CAP_MPSMAX(cap)     (((cap) >> 52) & 0xf)
+
+#define RT_NVME_VS(major, minor)    (((major) << 16) | ((minor) << 8))
+
+#define RT_NVME_AQ_DEPTH            32
+#define RT_NVME_NR_AEN_COMMANDS     1
+#define RT_NVME_AQ_BLK_MQ_DEPTH     (RT_NVME_AQ_DEPTH - RT_NVME_NR_AEN_COMMANDS)
+#define RT_NVME_AQ_MQ_TAG_DEPTH     (RT_NVME_AQ_BLK_MQ_DEPTH - 1)
+
+enum
+{
+    RT_NVME_CC_ENABLE           = 1 << 0,
+    RT_NVME_CC_CSS_NVM          = 0 << 4,
+    RT_NVME_CC_MPS_SHIFT        = 7,
+    RT_NVME_CC_ARB_RR           = 0 << 11,
+    RT_NVME_CC_ARB_WRRU         = 1 << 11,
+    RT_NVME_CC_ARB_VS           = 7 << 11,
+    RT_NVME_CC_SHN_NONE         = 0 << 14,
+    RT_NVME_CC_SHN_NORMAL       = 1 << 14,
+    RT_NVME_CC_SHN_ABRUPT       = 2 << 14,
+    RT_NVME_CC_SHN_MASK         = 3 << 14,
+    RT_NVME_CC_IOSQES           = 6 << 16,
+    RT_NVME_CC_IOCQES           = 4 << 20,
+    RT_NVME_CSTS_RDY            = 1 << 0,
+    RT_NVME_CSTS_CFS            = 1 << 1,
+    RT_NVME_CSTS_SHST_NORMAL    = 0 << 2,
+    RT_NVME_CSTS_SHST_OCCUR     = 1 << 2,
+    RT_NVME_CSTS_SHST_CMPLT     = 2 << 2,
+    RT_NVME_CSTS_SHST_MASK      = 3 << 2,
+};
+
+rt_packed(struct rt_nvme_id_power_state
+{
+    rt_le16_t   mp;         /* Maximum Power */
+    NVME_RSVD(1, 1);
+    rt_uint8_t  mxps_nops;  /* Max Power Scale, Non-Operational State */
+    rt_le32_t   enlat;      /* Entry Latency: microseconds */
+    rt_le32_t   exlat;      /* Exit Latency: microseconds */
+    rt_uint8_t  rrt;        /* Relative Read Throughput */
+    rt_uint8_t  rrl;        /* Relative Read Latency */
+    rt_uint8_t  rwt;        /* Relative Write Throughput */
+    rt_uint8_t  rwl;        /* Relative Write Latency */
+    rt_le16_t   idlp;       /* Idle Power */
+    rt_uint8_t  ips;        /* Idle Power Scale */
+    NVME_RSVD(19, 1);
+    rt_le16_t   actp;       /* Active Power */
+    rt_uint8_t  apw_aps;    /* Active Power Workload, Active Power Scale */
+    NVME_RSVD(23, 9);
+});
+
+rt_packed(struct rt_nvme_id_ctrl
+{
+    /* Controller Capabilities and Features */
+    rt_le16_t       vid;            /* PCI Vendor ID */
+    rt_le16_t       ssvid;          /* PCI Subsystem Vendor */
+    char            sn[20];         /* Serial Number */
+    char            mn[40];         /* Model Number */
+    char            fr[8];          /* Firmware Revision */
+    rt_uint8_t      rab;            /* Recommended Arbitration Burst */
+    rt_uint8_t      ieee[3];        /* IEEE OUI Identifier */
+    rt_uint8_t      mic;            /* Controller Multi-Path I/O and Namespace Sharing Capabilities */
+    rt_uint8_t      mdts;           /* Maximum Data Transfer Size */
+    rt_uint16_t     cntlid;         /* Controller ID */
+    rt_uint32_t     ver;            /* Version */
+    rt_uint32_t     rtd3r;          /* RTD3 Resume Latency */
+    rt_uint32_t     rtd3e;          /* RTD3 Entry Latency */
+    rt_uint32_t     oaes;           /* Optional Asynchronous Events Supported */
+#define RT_NVME_ID_CTRATT_ELBAS     15  /* Extended LBA Formats Supported */
+#define RT_NVME_ID_CTRATT_DNVMS     14  /* Delete NVM Set */
+#define RT_NVME_ID_CTRATT_DEG       13  /* Delete Endurance Group */
+#define RT_NVME_ID_CTRATT_VCM       12  /* Variable Capacity Management */
+#define RT_NVME_ID_CTRATT_FCM       11  /* Fixed Capacity Management */
+#define RT_NVME_ID_CTRATT_MDS       10  /* Multi-Domain Subsystem */
+#define RT_NVME_ID_CTRATT_UUIDL     9   /* UUID List */
+#define RT_NVME_ID_CTRATT_SQA       8   /* SQ Associations */
+#define RT_NVME_ID_CTRATT_NG        7   /* Namespace Granularity */
+#define RT_NVME_ID_CTRATT_TBKAS     6   /* Traffic Based Keep Alive Support */
+#define RT_NVME_ID_CTRATT_PLM       5   /* Predictable Latency Mode */
+#define RT_NVME_ID_CTRATT_EG        4   /* Endurance Groups */
+#define RT_NVME_ID_CTRATT_RRL       3   /* Read Recovery Levels */
+#define RT_NVME_ID_CTRATT_NVMS      2   /* NVM Sets */
+#define RT_NVME_ID_CTRATT_NOPSPM    1   /* Non-Operational Power State Permissive Mode */
+#define RT_NVME_ID_CTRATT_HIS       0   /* Host Identifier Support */
+    rt_uint32_t     ctratt;         /* Controller Attributes */
+    rt_uint16_t     rrls;           /* Read Recovery Levels Supported */
+    NVME_RSVD(102, 9);
+    rt_uint8_t      cntrltype;      /* Controller Type */
+    rt_uint8_t      fguid[16];      /* FRU Globally Unique Identifier */
+    rt_uint16_t     crdt1;          /* Command Retry Delay Time 1 */
+    rt_uint16_t     crdt2;          /* Command Retry Delay Time 2 */
+    rt_uint16_t     crdt3;          /* Command Retry Delay Time 3 */
+    NVME_RSVD(134, 119);
+#define RT_NVME_ID_NVMSR_NVMEE      1   /* NVMe Enclosure */
+#define RT_NVME_ID_NVMSR_NVMESD     0   /* NVMe Storage Device */
+    rt_uint8_t      nvmsr;          /* NVM Subsystem Report */
+
+#define RT_NVME_ID_VWCI_VWCRV       7   /* VPD Write Cycles Remaining Valid */
+#define RT_NVME_ID_VWCI_VWCR        0   /* VPD Write Cycles Remaining */
+    rt_uint8_t      vwci;           /* VPD Write Cycle Information */
+#define RT_NVME_ID_MEC_PCIEME       1   /* PCIe Port Management Endpoint */
+#define RT_NVME_ID_MEC_SMBUSME      0   /* SMBus/I2C Port Management Endpoint */
+    rt_uint8_t      mec;            /* Management Endpoint Capabilities  */
+
+    /* Admin Command Set Attributes & Optional Controller Capabilities */
+    rt_le16_t       oacs;           /* Optional Admin Command Support */
+    rt_uint8_t      acl;            /* Abort Command Limit */
+    rt_uint8_t      aerl;           /* Asynchronous Event Request Limit */
+#define RT_NVME_ID_FRMW_SMUD        5   /* Support Multiple Update Detection */
+#define RT_NVME_ID_FRMW_FAWR        4   /* Firmware Activation Without Reset */
+#define RT_NVME_ID_FRMW_NOFS        1   /* Number Of Firmware Slots */
+#define RT_NVME_ID_FRMW_FFSRO       0   /* First Firmware Slot Read Only */
+    rt_uint8_t      frmw;           /* Firmware Updates */
+    rt_uint8_t      lpa;            /* Log Page Attributes */
+    rt_uint8_t      elpe;           /* Error Log Page Entries */
+    rt_uint8_t      npss;           /* Number of Power States Support */
+    rt_uint8_t      avscc;          /* Admin Vendor Specific Command Configuration */
+    rt_uint8_t      apsta;          /* Autonomous Power State Transition Attributes */
+    rt_le16_t       wctemp;         /* Warning Composite Temperature Threshold */
+    rt_le16_t       cctemp;         /* Critical Composite Temperature Threshold */
+    rt_uint16_t     mtfa;           /* Maximum Time for Firmware Activation */
+    rt_uint32_t     hmpre;          /* Host Memory Buffer Preferred Size */
+    rt_uint32_t     hmmin;          /* Host Memory Buffer Minimum Size */
+    rt_uint8_t      tnvmcap[16];    /* Total NVM Capacity */
+    rt_uint8_t      unvmcap[16];    /* Unallocated NVM Capacity */
+#define RT_NVME_ID_RPMBS_ASZ        24  /* Access Size */
+#define RT_NVME_ID_RPMBS_TSZ        16  /* Total Size */
+#define RT_NVME_ID_RPMBS_AM         3   /* Authentication Method */
+#define RT_NVME_ID_RPMBS_NORPMBU    2   /* Number of RPMB Units */
+    rt_uint32_t     rpmbs;          /* Replay Protected Memory Block Support */
+    rt_uint16_t     edstt;          /* Extended Device Self-test Time */
+    rt_uint8_t      dsto;           /* Device Self-test Options */
+    rt_uint8_t      fwug;           /* Firmware Update Granularity */
+    rt_uint16_t     kas;            /* Keep Alive Support */
+    rt_uint16_t     hctma;          /* Host Controlled Thermal Management Attributes */
+    rt_uint16_t     mntmt;          /* Minimum Thermal Management Temperature */
+    rt_uint16_t     mxtmt;          /* Maximum Thermal Management Temperature */
+#define RT_NVME_ID_SANICAP_NODMMAS  30  /* No-Deallocate Modifies Media After Sanitize */
+#define RT_NVME_ID_SANICAP_NDI      29  /* No-Deallocate Inhibited */
+#define RT_NVME_ID_SANICAP_OWS      2   /* Overwrite Support */
+#define RT_NVME_ID_SANICAP_BES      1   /* Block Erase Support */
+#define RT_NVME_ID_SANICAP_CES      0   /* Crypto Erase Support */
+    rt_uint32_t     sanicap;        /* Sanitize Capabilities */
+    rt_uint32_t     hmminds;        /* Host Memory Buffer Minimum Descriptor Entry Size */
+    rt_uint16_t     hmmaxd;         /* Host Memory Maximum Descriptors Entries */
+    rt_uint16_t     nsetidmax;      /* NVM Set Identifier Maximum */
+    rt_uint16_t     endgidmax;      /* Endurance Group Identifier Maximum */
+    rt_uint8_t      anatt;          /* ANA Transition Time */
+    rt_uint8_t      anacap;         /* Asymmetric Namespace Access Capabilities */
+    rt_uint32_t     anagrpmax;      /* ANA Group Identifier Maximum */
+    rt_uint32_t     nanagrpid;      /* Number of ANA Group Identifiers */
+    rt_uint32_t     pels;           /* Persistent Event Log Size */
+    rt_uint16_t     dmid;           /* Domain Identifier */
+    NVME_RSVD(358, 10);
+    rt_uint8_t      megcap[16];     /* Max Endurance Group Capacity */
+    NVME_RSVD(384, 128);
+
+    /* NVM Command Set Attributes */
+    rt_uint8_t      sqes;           /* Submission Queue Entry Size */
+    rt_uint8_t      cqes;           /* Completion Queue Entry Size */
+    rt_le16_t       maxcmd;         /* Maximum Outstanding Commands */
+    rt_le32_t       nn;             /* Number of Namespaces */
+    rt_le16_t       oncs;           /* Optional NVM Command Support */
+    rt_le16_t       fuses;          /* Fused Operation Support */
+    rt_uint8_t      fna;            /* Format NVM Attributes */
+    rt_uint8_t      vwc;            /* Volatile Write Cache */
+    rt_le16_t       awun;           /* Atomic Write Unit Normal */
+    rt_le16_t       awupf;          /* Atomic Write Unit Power Fail */
+    rt_uint8_t      nvscc;          /* I/O Command Set Vendor Specific Command Configuration */
+    rt_uint8_t      nwpc;           /* Namespace Write Protection Capabilities */
+    rt_le16_t       acwu;           /* Atomic Compare & Write Unit */
+    rt_le16_t       cdfs;           /* Copy Descriptor Formats Supported */
+#define RT_NVME_ID_SGL_SUPPORT_MASK 0x3
+    rt_le32_t       sgls;           /* SGL Support */
+    rt_uint32_t     mnan;           /* Maximum Number of Allowed Namespaces */
+    char            maxdna[16];     /* Maximum Domain Namespace Attachments */
+    rt_le32_t       maxcna;         /* Maximum I/O Controller Namespace Attachments */
+    NVME_RSVD(564, 204);
+    rt_uint8_t      subnqn[256];    /* NVM Subsystem NVMe Qualified Name */
+    NVME_RSVD(1024, 768);
+    rt_le32_t       ioccsz;         /* I/O Queue Command Capsule Supported Size */
+    rt_le32_t       iorcsz;         /* I/O Queue Response Capsule Supported Size */
+    rt_le16_t       icdoff;         /* In Capsule Data Offset */
+    rt_uint8_t      ctrattr;        /* Fabrics Controller Attributes */
+    rt_uint8_t      msdbd;          /* Maximum SGL Data Block Descriptors */
+    rt_le16_t       ofcs;           /* Optional Fabric Commands Support */
+    rt_uint8_t      dctype;
+    NVME_RSVD(1807, 241);
+
+    /* Power State Descriptors */
+    struct rt_nvme_id_power_state psd[32];
+
+    /* Vendor Specific */
+    rt_uint8_t      vs[1024];
+});
+
+enum
+{
+    RT_NVME_CTRL_CMIC_MULTI_PORT                = 1 << 0,
+    RT_NVME_CTRL_CMIC_MULTI_CTRL                = 1 << 1,
+    RT_NVME_CTRL_CMIC_ANA                       = 1 << 3,
+    RT_NVME_CTRL_ONCS_COMPARE                   = 1 << 0,
+    RT_NVME_CTRL_ONCS_WRITE_UNCORRECTABLE       = 1 << 1,
+    RT_NVME_CTRL_ONCS_DSM                       = 1 << 2,
+    RT_NVME_CTRL_ONCS_WRITE_ZEROES              = 1 << 3,
+    RT_NVME_CTRL_ONCS_RESERVATIONS              = 1 << 5,
+    RT_NVME_CTRL_ONCS_TIMESTAMP                 = 1 << 6,
+    RT_NVME_CTRL_VWC_PRESENT                    = 1 << 0,
+    RT_NVME_CTRL_OACS_SEC_SUPP                  = 1 << 0,
+    RT_NVME_CTRL_OACS_NS_MNGT_SUPP              = 1 << 3,
+    RT_NVME_CTRL_OACS_DIRECTIVES                = 1 << 5,
+    RT_NVME_CTRL_OACS_DBBUF_SUPP                = 1 << 8,
+    RT_NVME_CTRL_LPA_CMD_EFFECTS_LOG            = 1 << 1,
+    RT_NVME_CTRL_CTRATT_128_ID                  = 1 << 0,
+    RT_NVME_CTRL_CTRATT_NON_OP_PSP              = 1 << 1,
+    RT_NVME_CTRL_CTRATT_NVM_SETS                = 1 << 2,
+    RT_NVME_CTRL_CTRATT_READ_RECV_LVLS          = 1 << 3,
+    RT_NVME_CTRL_CTRATT_ENDURANCE_GROUPS        = 1 << 4,
+    RT_NVME_CTRL_CTRATT_PREDICTABLE_LAT         = 1 << 5,
+    RT_NVME_CTRL_CTRATT_NAMESPACE_GRANULARITY   = 1 << 7,
+    RT_NVME_CTRL_CTRATT_UUID_LIST               = 1 << 9,
+};
+
+struct rt_nvme_lba_format
+{
+    rt_le16_t   ms;         /* Metadata size */
+    rt_uint8_t  ds;         /* Data size */
+    rt_uint8_t  rp;         /* Relative performance */
+};
+
+rt_packed(struct rt_nvme_id_ns
+{
+    rt_le64_t   nsze;       /* Namespace size */
+    rt_le64_t   ncap;       /* Namespace capacity */
+    rt_le64_t   nuse;       /* Namespace utilization */
+    rt_uint8_t  nsfeat;     /* Namespace features */
+    rt_uint8_t  nlbaf;      /* Number of lba formats */
+    rt_uint8_t  flbas;      /* Formatted lba size */
+    rt_uint8_t  mc;         /* Metadata capabilities */
+    rt_uint8_t  dpc;        /* End-to-end data protection capabilities */
+    rt_uint8_t  dps;        /* End-to-end data protection type settings */
+    rt_uint8_t  nmic;       /* Namespace Multi-path I/O and Namespace Sharing Capabilities */
+    rt_uint8_t  rescap;     /* Reservation Capabilities */
+    rt_uint8_t  fpi;        /* Format Progress Indicator */
+    rt_uint8_t  dlfeat;     /* Deallocate Logical Block Features */
+    rt_le16_t   nawun;      /* Namespace Atomic Write Unit Normal  */
+    rt_le16_t   nawupf;     /* Namespace Atomic Write Unit Power Fail */
+    rt_le16_t   nacwu;      /* Namespace Atomic Compare & Write Unit */
+    rt_le16_t   nabsn;      /* Namespace Atomic Boundary Size Normal */
+    rt_le16_t   nabo;       /* Namespace Atomic Boundary Offset */
+    rt_le16_t   nabspf;     /* Namespace Atomic Boundary Size Power Fail */
+    rt_uint16_t noiob;      /* Namespace Optimal IO Boundary */
+    rt_le64_t   nvmcap[2];  /* NVMe Capacity */
+    rt_uint16_t npwg;       /* Namespace Preferred Write Granularity  */
+    rt_uint16_t npwa;       /* Namespace Preferred Write Alignment */
+    rt_uint16_t npdg;       /* Namespace Preferred Deallocate Granularity */
+    rt_uint16_t npda;       /* Namespace Preferred Deallocate Alignment */
+    rt_uint16_t nows;       /* Namespace Optimal Write Size */
+    NVME_RSVD(118, 18);
+    rt_uint32_t anagrpid;   /* ANA Group Identifier */
+    NVME_RSVD(139, 3);
+    rt_uint8_t  nsattr;     /* Namespace Attributes */
+    rt_uint16_t nvmsetid;   /* NVMe Set Identifier */
+    rt_uint16_t endgid;     /* Endurance Group Identifier */
+    rt_uint8_t  nguid[16];  /* Namespace Globally Unique Identifier */
+    rt_uint8_t  eui64[8];   /* IEEE Extended Unique Identifier */
+
+    /* Logical Block Address Format */
+    struct rt_nvme_lba_format lbaf[16];
+    NVME_RSVD(171, 192);
+
+    /* Vendor specific */
+    rt_uint8_t  vs[3712];
+});
+
+enum
+{
+    RT_NVME_NS_FEAT_THIN        = 1 << 0,
+    RT_NVME_NS_FLBAS_LBA_MASK   = 0xf,
+    RT_NVME_NS_FLBAS_LBA_UMASK  = 0x60,
+    RT_NVME_NS_FLBAS_LBA_SHIFT  = 1,
+    RT_NVME_NS_FLBAS_META_EXT   = 0x10,
+    RT_NVME_LBAF_RP_BEST        = 0,
+    RT_NVME_LBAF_RP_BETTER      = 1,
+    RT_NVME_LBAF_RP_GOOD        = 2,
+    RT_NVME_LBAF_RP_DEGRADED    = 3,
+    RT_NVME_NS_DPC_PI_LAST      = 1 << 4,
+    RT_NVME_NS_DPC_PI_FIRST     = 1 << 3,
+    RT_NVME_NS_DPC_PI_TYPE3     = 1 << 2,
+    RT_NVME_NS_DPC_PI_TYPE2     = 1 << 1,
+    RT_NVME_NS_DPC_PI_TYPE1     = 1 << 0,
+    RT_NVME_NS_DPS_PI_FIRST     = 1 << 3,
+    RT_NVME_NS_DPS_PI_MASK      = 0x7,
+    RT_NVME_NS_DPS_PI_TYPE1     = 1,
+    RT_NVME_NS_DPS_PI_TYPE2     = 2,
+    RT_NVME_NS_DPS_PI_TYPE3     = 3,
+};
+
+struct rt_nvme_ops;
+struct rt_nvme_controller;
+
+/*
+ * An NVM Express queue. Each device has at least two (one for admin commands
+ * and one for I/O commands).
+ */
+struct rt_nvme_queue
+{
+    struct rt_nvme_controller *nvme;
+    struct rt_nvme_command *sq_cmds;
+    struct rt_nvme_completion *cq_entry;
+
+    rt_ubase_t sq_cmds_phy;
+    rt_ubase_t cq_entry_phy;
+
+    rt_uint32_t *doorbell;
+    rt_uint16_t qid;
+    rt_uint16_t depth;
+    rt_uint16_t sq_head;
+    rt_uint16_t sq_tail;
+    rt_uint16_t cq_head;
+    rt_uint16_t cq_phase;
+
+    rt_err_t err;
+    struct rt_nvme_command *cmd;
+
+    struct rt_completion done;
+    struct rt_spinlock lock;
+};
+
+struct rt_nvme_controller
+{
+    rt_list_t list;
+    struct rt_device *dev;
+
+    int nvme_id;
+    char name[RT_NAME_MAX];
+
+    void *regs;
+    rt_uint64_t cap;
+    rt_uint32_t page_shift;
+    rt_uint32_t page_size;
+    rt_uint32_t queue_depth;
+    rt_uint32_t io_queue_max;
+    rt_uint32_t ctrl_config;
+    rt_uint32_t max_transfer_shift:8;
+    rt_uint32_t volatile_write_cache:8;
+    rt_uint32_t write_zeroes:1;
+    rt_uint32_t sgl_mode:2;
+    rt_uint32_t doorbell_stride;
+    rt_uint32_t *doorbell_tbl;
+
+    const struct rt_nvme_ops *ops;
+
+#define RT_USING_NVME_QUEUE (1 + (RT_USING_NVME_IO_QUEUE * RT_CPUS_NR))
+    int irqs_nr;
+    int irqs[RT_USING_NVME_QUEUE];
+    union
+    {
+        struct
+        {
+            struct rt_nvme_queue admin_queue;
+            struct rt_nvme_queue io_queues[RT_USING_NVME_IO_QUEUE * RT_CPUS_NR];
+        };
+        struct rt_nvme_queue queue[RT_USING_NVME_QUEUE];
+    };
+
+    volatile rt_atomic_t cmdid;
+    volatile rt_atomic_t ioqid[RT_CPUS_NR];
+
+    rt_list_t ns_nodes;
+};
+
+struct rt_nvme_device
+{
+    struct rt_blk_disk parent;
+    struct rt_nvme_controller *ctrl;
+
+    rt_list_t list;
+
+    rt_uint32_t nsid;
+    rt_uint32_t lba_shift;
+    struct rt_nvme_id_ns id;
+};
+#define rt_disk_to_nvme_device(disk) rt_container_of(disk, struct rt_nvme_device, parent)
+
+struct rt_nvme_ops
+{
+    const char *name;
+
+    /* Controller-specific NVM Express queue setup */
+    rt_err_t (*setup_queue)(struct rt_nvme_queue *queue);
+    /* Controller-specific NVM Express queue cleanup */
+    rt_err_t (*cleanup_queue)(struct rt_nvme_queue *queue);
+    /* Controller-specific NVM Express command submission */
+    rt_err_t (*submit_cmd)(struct rt_nvme_queue *queue, struct rt_nvme_command *cmd);
+    /* Controller-specific NVM Express command completion */
+    void (*complete_cmd)(struct rt_nvme_queue *queue, struct rt_nvme_command *cmd);
+};
+
+rt_err_t rt_nvme_controller_register(struct rt_nvme_controller *nvme);
+rt_err_t rt_nvme_controller_unregister(struct rt_nvme_controller *nvme);
+
+#endif /* __NVME_H__ */
diff --git a/components/drivers/include/rtdevice.h b/components/drivers/include/rtdevice.h
index 19e9de0467..18a64e259c 100644
--- a/components/drivers/include/rtdevice.h
+++ b/components/drivers/include/rtdevice.h
@@ -55,6 +55,10 @@ extern "C" {
 
 #include "drivers/iio.h"
 
+#ifdef RT_USING_NVME
+#include "drivers/nvme.h"
+#endif
+
 #ifdef RT_USING_OFW
 #include "drivers/ofw.h"
 #include "drivers/ofw_fdt.h"