how to implement a customized engine

[wangyihanlarry]

Hi all,

I tried to write my own helloworld engine by refering the code of null.c as below

#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>

#include "../fio.h"

#ifdef FIO_EXTERNAL_ENGINE

struct HelloWorldData {
    HelloWorldData(struct thread_data *td)
    {
        io_us = nullptr;
        queued = 0;
        events = 0;
        fd = -1;

        printf("helloworld_init: iodepth=%d\n", td->o.iodepth);

        if (td->o.iodepth != 1) {
            io_us = (struct io_u **)calloc(td->o.iodepth, sizeof(struct io_u *));
            td->io_ops->flags |= FIO_ASYNCIO_SETS_ISSUE_TIME;
        } else {
            td->io_ops->flags |= FIO_SYNCIO;
        }

        td_set_ioengine_flags(td);
    }

    ~HelloWorldData()
    {
        printf("helloworld_cleanup\n");
        if (fd >= 0)
            close(fd);
        free(io_us);
    }

    static HelloWorldData *get(struct thread_data *td)
    {
        return reinterpret_cast<HelloWorldData *>(td->io_ops_data);
    }

    io_u *helloworld_event(struct thread_data *, int event)
    {
        printf("helloworld_event: event=%d\n", event);
        return io_us[event];
    }

    int helloworld_getevents(struct thread_data *, unsigned int min_events,
                             unsigned int max, const struct timespec *t)
    {
        int ret = 0;

        printf("helloworld_getevents: min_events=%u, max=%u\n", min_events, max);

        if (min_events) {
            ret = events;
            events = 0;
        }

        printf("helloworld_getevents: returning %d events\n", ret);
        return ret;
    }

    void helloworld_queued(struct thread_data *td)
    {
        struct timespec now;

        printf("helloworld_queued: queued=%d\n", queued);

        if (!fio_fill_issue_time(td)) {
            printf("helloworld_queued: fio_fill_issue_time failed\n");
            return;
        }

        // fio_gettime(&now, NULL);

        for (int i = 0; i < queued; i++) {
            struct io_u *io_u = io_us[i];

            memcpy(&io_u->issue_time, &now, sizeof(now));
            io_u_queued(td, io_u);
        }
    }

    int helloworld_commit(struct thread_data *td)
    {
        printf("helloworld_commit: queued=%d, events=%d\n", queued, events);
        if (!events) {
            helloworld_queued(td);

#ifndef FIO_EXTERNAL_ENGINE
            io_u_mark_submit(td, queued);
#endif
            for (int i = 0; i < queued; i++) {
                struct io_u *io_u = io_us[i];
                printf("helloworld_commit: writing %llu bytes\n", (unsigned long long)io_u->xfer_buflen);
                printf("helloworld_commit: content=%.*s\n", (int)io_u->xfer_buflen, (char *)io_u->buf);
                if (write(fd, io_u->buf, io_u->xfer_buflen) != io_u->xfer_buflen) {
                    perror("write");
                    return -1;
                }
            }
            events = queued;
            queued = 0;
        }

        return 0;
    }

    fio_q_status helloworld_queue(struct thread_data *td, struct io_u *io_u)
    {
        fio_ro_check(td, io_u);

        printf("helloworld_queue: queued=%d, events=%d\n", queued, events);

        if (td->io_ops->flags & FIO_SYNCIO) {
            printf("helloworld_queue: FIO_SYNCIO flag is set, completing IO immediately\n");
            return FIO_Q_COMPLETED;
        }
        if (events) {
            printf("helloworld_queue: events are pending, returning FIO_Q_BUSY\n");
            return FIO_Q_BUSY;
        }

        io_us[queued++] = io_u;
        printf("helloworld_queue: added io_u to queue, queued=%d\n", queued);
        return FIO_Q_QUEUED;
    }

    int helloworld_open(struct thread_data *, struct fio_file *f)
    {
        printf("helloworld_open: opening file %s\n", f->file_name);
        fd = open(f->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
        if (fd < 0) {
            perror("open");
            return -1;
        }
        return 0;
    }

    struct io_u **io_us;
    int queued;
    int events;
    int fd;
};

extern "C" {

static struct io_u *helloworld_event(struct thread_data *td, int event)
{
    return HelloWorldData::get(td)->helloworld_event(td, event);
}

static int helloworld_getevents(struct thread_data *td, unsigned int min_events,
                                unsigned int max, const struct timespec *t)
{
    return HelloWorldData::get(td)->helloworld_getevents(td, min_events, max, t);
}

static int helloworld_commit(struct thread_data *td)
{
    return HelloWorldData::get(td)->helloworld_commit(td);
}

static fio_q_status helloworld_queue(struct thread_data *td, struct io_u *io_u)
{
    return HelloWorldData::get(td)->helloworld_queue(td, io_u);
}

static int helloworld_open(struct thread_data *td, struct fio_file *f)
{
    return HelloWorldData::get(td)->helloworld_open(td, f);
}

static int helloworld_init(struct thread_data *td)
{
    td->io_ops_data = new HelloWorldData(td);
    return 0;
}

static void helloworld_cleanup(struct thread_data *td)
{
    delete HelloWorldData::get(td);
}

static struct ioengine_ops ioengine;
void get_ioengine(struct ioengine_ops **ioengine_ptr)
{
    *ioengine_ptr = &ioengine;

    ioengine.name           = "cpp_helloworld";
    ioengine.version        = FIO_IOOPS_VERSION;
    ioengine.queue          = helloworld_queue;
    ioengine.commit         = helloworld_commit;
    ioengine.getevents      = helloworld_getevents;
    ioengine.event          = helloworld_event;
    ioengine.init           = helloworld_init;
    ioengine.cleanup        = helloworld_cleanup;
    ioengine.open_file      = helloworld_open;
    ioengine.flags          = 0;
}
}

#endif /* FIO_EXTERNAL_ENGINE */

but I met the following issues when compilng it

engines % g++ -O2 -g -shared -rdynamic -fPIC -o cpp_hellowold helloworld.cpp -include ../config-host.h -DFIO_EXTERNAL_ENGINE
Undefined symbols for architecture arm64:
"io_u_queued(thread_data*, io_u*)", referenced from:
HelloWorldData::helloworld_commit(thread_data*) in helloworld-4e8e11.o
ld: symbol(s) not found for architecture arm64
clang: error: linker command failed with exit code 1 (use -v to see invocation)

[vincentkfu]

io_u_queued() is declared in io_u.h, so try adding #include "../io_u.h"

[wangyihanlarry]

I've made some progress today, I've refactor the code as following,

#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>
#include "../io_u.h"
#include "../fio.h"

#ifdef __cplusplus

#ifdef FIO_EXTERNAL_ENGINE

struct HelloWorldData {
    HelloWorldData(struct thread_data *td)
    {
        io_us = nullptr;
        queued = 0;
        events = 0;
        fd = -1;

        printf("helloworld_init: iodepth=%d\n", td->o.iodepth);

        if (td->o.iodepth != 1) {
            io_us = (struct io_u **)calloc(td->o.iodepth, sizeof(struct io_u *));
            td->io_ops->flags |= FIO_ASYNCIO_SETS_ISSUE_TIME;
        } else {
            td->io_ops->flags |= FIO_SYNCIO;
        }

        td_set_ioengine_flags(td);
    }

    ~HelloWorldData()
    {
        printf("helloworld_cleanup\n");
        if (fd >= 0)
            close(fd);
        free(io_us);
    }

    static HelloWorldData *get(struct thread_data *td)
    {
        return reinterpret_cast<HelloWorldData *>(td->io_ops_data);
    }

    io_u *helloworld_event(struct thread_data *, int event)
    {
        printf("helloworld_event: event=%d\n", event);
        return io_us[event];
    }

    int helloworld_getevents(struct thread_data *, unsigned int min_events,
                             unsigned int max, const struct timespec *t)
    {
        int ret = 0;

        printf("helloworld_getevents: min_events=%u, max=%u\n", min_events, max);

        if (min_events) {
            ret = events;
            events = 0;
        }

        printf("helloworld_getevents: returning %d events\n", ret);
        return ret;
    }

    void helloworld_queued(struct thread_data *td)
    {
        struct timespec now;

        printf("helloworld_queued: queued=%d\n", queued);

        if (!fio_fill_issue_time(td)) {
            printf("helloworld_queued: fio_fill_issue_time failed\n");
            return;
        }

        // fio_gettime(&now, NULL);

        for (int i = 0; i < queued; i++) {
            struct io_u *io_u = io_us[i];

            memcpy(&io_u->issue_time, &now, sizeof(now));
            io_u_queued(td, io_u);
        }
    }

    int helloworld_commit(struct thread_data *td)
    {
        printf("helloworld_commit: queued=%d, events=%d\n", queued, events);
        if (!events) {
            helloworld_queued(td);

#ifndef FIO_EXTERNAL_ENGINE
            io_u_mark_submit(td, queued);
#endif
            for (int i = 0; i < queued; i++) {
                struct io_u *io_u = io_us[i];
                printf("helloworld_commit: writing %llu bytes\n", (unsigned long long)io_u->xfer_buflen);
                printf("helloworld_commit: content=%.*s\n", (int)io_u->xfer_buflen, (char *)io_u->buf);
                if (write(fd, io_u->buf, io_u->xfer_buflen) != io_u->xfer_buflen) {
                    perror("write");
                    return -1;
                }
            }
            events = queued;
            queued = 0;
        }

        return 0;
    }

    fio_q_status helloworld_queue(struct thread_data *td, struct io_u *io_u)
    {
        fio_ro_check(td, io_u);

        printf("helloworld_queue: queued=%d, events=%d\n", queued, events);

        if (td->io_ops->flags & FIO_SYNCIO) {
            printf("helloworld_queue: FIO_SYNCIO flag is set, completing IO immediately\n");
            return FIO_Q_COMPLETED;
        }
        if (events) {
            printf("helloworld_queue: events are pending, returning FIO_Q_BUSY\n");
            return FIO_Q_BUSY;
        }

        io_us[queued++] = io_u;
        printf("helloworld_queue: added io_u to queue, queued=%d\n", queued);
        return FIO_Q_QUEUED;
    }

    int helloworld_open(struct thread_data *, struct fio_file *f)
    {
        printf("helloworld_open: opening file %s\n", f->file_name);
        fd = open(f->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
        if (fd < 0) {
            perror("open");
            return -1;
        }
        return 0;
    }

    struct io_u **io_us;
    int queued;
    int events;
    int fd;
};

extern "C" {

static struct io_u *helloworld_event(struct thread_data *td, int event)
{
    return HelloWorldData::get(td)->helloworld_event(td, event);
}

static int helloworld_getevents(struct thread_data *td, unsigned int min_events,
                                unsigned int max, const struct timespec *t)
{
    return HelloWorldData::get(td)->helloworld_getevents(td, min_events, max, t);
}

static int helloworld_commit(struct thread_data *td)
{
    return HelloWorldData::get(td)->helloworld_commit(td);
}

static fio_q_status helloworld_queue(struct thread_data *td, struct io_u *io_u)
{
    return HelloWorldData::get(td)->helloworld_queue(td, io_u);
}

static int helloworld_open(struct thread_data *td, struct fio_file *f)
{
    return HelloWorldData::get(td)->helloworld_open(td, f);
}

static int helloworld_init(struct thread_data *td)
{
    td->io_ops_data = new HelloWorldData(td);
    return 0;
}

static void helloworld_cleanup(struct thread_data *td)
{
    delete HelloWorldData::get(td);
}

static struct ioengine_ops ioengine;
void get_ioengine(struct ioengine_ops **ioengine_ptr)
{
    *ioengine_ptr = &ioengine;

    ioengine.name           = "cpp_helloworld";
    ioengine.version        = FIO_IOOPS_VERSION;
    ioengine.queue          = helloworld_queue;
    ioengine.commit         = helloworld_commit;
    ioengine.getevents      = helloworld_getevents;
    ioengine.event          = helloworld_event;
    ioengine.init           = helloworld_init;
    ioengine.cleanup        = helloworld_cleanup;
    ioengine.open_file      = helloworld_open;
    ioengine.flags          = 0;
}
}

#endif /* FIO_EXTERNAL_ENGINE */

#endif /* __cplusplus */

them I complied it with following command, it works, but by executing still reports errors, in below, any thing wrong?

# COMPLIE THE CODE
g++ -O2 -g -shared -rdynamic -fPIC -o cpp_helloworld helloworld.c -include ../config-host.h -DFIO_EXTERNAL_ENGINE

# EXECUTING THE ENGINE
LD_LIBRARY_PATH=./engines ./fio --directory=/tmp/file --name=randwrite --ioengine=cpp_helloworld --verify=pattern --verify_pattern=0xabcd --size=100g --rw=write --bs=4k --iodepth=2 --size=512M --time_based --runtime=5
fio: verification read phase will never start because write phase uses all of runtime
randwrite: (g=0): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=cpp_helloworld, iodepth=2
fio-3.36
Starting 1 process
randwrite: Laying out IO file (1 file / 512MiB)
helloworld_init: iodepth=2
helloworld_open: opening file /tmp/file/randwrite.0.0
fio: cache invalidation of /tmp/file/randwrite.0.0 failed: Bad file descriptor
helloworld_queue: queued=0, events=0
helloworld_queue: added io_u to queue, queued=1
helloworld_commit: queued=1, events=0
helloworld_queued: queued=1
./fio: symbol lookup error: ./engines/cpp_helloworld: undefined symbol: _Z11io_u_queuedP11thread_dataP4io_u

randwrite: (groupid=0, jobs=1): err=127 (): pid=21987: Sat Jul 27 00:02:55 2024
  cpu          : usr=0.00%, sys=0.00%, ctx=0, majf=0, minf=0
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,1,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=2

Run status group 0 (all jobs):

Disk stats (read/write):
  vda: ios=0/2, sectors=0/16, merge=0/0, ticks=0/1, in_queue=0, util=0.81%

# CHECK GCC VERSION
gcc --version
gcc (GCC) 8.3.1 20190311 (Red Hat 8.3.1-3)
Copyright (C) 2018 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.