class KernelDevice : public BlockDevice {
	int fd_direct, fd_buffered; // 分别存放以direct和buffered两种方式打开裸设备时的fd
	uint64_t size; // 设备总的大小
	uint64_t block_size; // 块的大小
	std::string path; // 路径
	bool aio, dio;

	// libaio相关的线程
	struct AioCompletionThread : public Thread {
		KernelDevice *bdev;
		explicit AioCompletionThread(KernelDevice *b) : bdev(b) {}
		void *entry() override {
			bdev->_aio_thread();
			return NULL;
		}
	} aio_thread;

	// aio操作相关的队列和回调函数
	aio_queue_t aio_queue;
	aio_callback_t aio_callback;
	void *aio_callback_priv;
	bool aio_stop;
};

BlockDevice *BlockDevice::create(CephContext* cct, const string& path,
		aio_callback_t cb, void *cbpriv) {

	// 通过设备的path，判断出设备的类型
	string type = "kernel";
	char buf[PATH_MAX + 1];
	int r = ::readlink(path.c_str(), buf, sizeof(buf) - 1);
	if (r >= 0) {
		buf[r] = '\0';
		char *bname = ::basename(buf);
		if (strncmp(bname, SPDK_PREFIX, sizeof(SPDK_PREFIX)-1) == 0)
			type = "ust-nvme";
	}

	if (type == "kernel") {
		return new KernelDevice(cct, cb, cbpriv); // kernel
	}
	......
	if (type == "ust-nvme") {
		return new NVMEDevice(cct, cb, cbpriv); // nvme
	}
	......
}

int KernelDevice::open(const string& p)
{
	// 分别以fd_direct/fd_buffered方式打开块设备
	fd_direct = ::open(path.c_str(), O_RDWR | O_DIRECT);
	fd_buffered = ::open(path.c_str(), O_RDWR);

	// 读取block size等参数
	block_size = cct->_conf->bdev_block_size;
	......

	// 如果是aio，初始化aio相关参数，并启动aio线程
	r = _aio_start();
	......
}

struct IOContext {
	private:
		std::mutex lock;
		std::condition_variable cond;

	public:
		void *priv;

		std::list<aio_t> pending_aios;    // 待执行的aio
		std::list<aio_t> running_aios;    // 正在执行的aio

		// 计数
		std::atomic_int num_pending = {0};
		std::atomic_int num_running = {0};
};

struct aio_t {
	struct iocb iocb; // libaio相关的结构体
	......
	void pwritev(uint64_t _offset, uint64_t len) {
		offset = _offset;
		length = len;
		io_prep_pwritev(&iocb, fd, &iov[0], iov.size(), offset); // 准备数据
	}
	......
};

int KernelDevice::aio_write(uint64_t off, bufferlist &bl, IOContext *ioc, bool buffered)
{
	......
	if (aio && dio && !buffered) {
		ioc->pending_aios.push_back(aio_t(ioc, fd_direct)); // 放入IOContext的pending队列，等待执行
		++ioc->num_pending;

		// 将待写入的数据准备在aio的buffer中
		aio_t& aio = ioc->pending_aios.back();
		bl.prepare_iov(&aio.iov);
		for (unsigned i=0; i<aio.iov.size(); ++i) {
			aio.bl.claim_append(bl);
			aio.pwritev(off, len); // 写buffer
		}
	}
}

void KernelDevice::aio_submit(IOContext *ioc)
{
	if (ioc->num_pending.load() == 0) {
		return;
	}

	// 获取pending的aio
	list<aio_t>::iterator e = ioc->running_aios.begin();
	ioc->running_aios.splice(e, ioc->pending_aios);
	......

	// 批量提交aio
	r = aio_queue.submit_batch(ioc->running_aios.begin(), e, 
			ioc->num_running.load(), priv, &retries);
}

int aio_queue_t::submit_batch(aio_iter begin, aio_iter end, 
		uint16_t aios_size, void *priv, 
		int *retries)
{
	......
	while (left > 0) {
		int r = io_submit(ctx, left, piocb + done); // 调用libaio相关的api提交io
	}
	......
}

void KernelDevice::_aio_thread()
{
	......
	while (!aio_stop) {
		int r = aio_queue.get_next_completed(cct->_conf->bdev_aio_poll_ms, // 调用libaio相关的api，检查io是否完成
				aio, max);

		if (ioc->priv) {
			if (--ioc->num_running == 0) {
				aio_callback(aio_callback_priv, ioc->priv); // 执行回调
			}
		}
	}
	......
}

int aio_queue_t::get_next_completed(int timeout_ms, aio_t **paio, int max)
{
	......
	do {
		r = io_getevents(ctx, 1, max, event, &t); // 调用libaio相关的api，获取已经完成aio请求
	} while (r == -EINTR);
}