Skip to content

Commit

Permalink
disk: parameterize lmdb map size (urbit#655)
Browse files Browse the repository at this point in the history 
Adds `--lmdb-map-size` to specify LMDB's initial memory mapping size,
intended for use by users who need a size smaller than our default of
500G. Note: `case 12` to avoid collision with the other arg added in
urbit#654.
  • Loading branch information
@pkova
pkova authored May 29, 2024
2 parents 6c07292 + 41fa70e commit 51b2e47
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 43 deletions.
21 changes: 6 additions & 15 deletions pkg/vere/disk.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,6 @@ struct _u3_disk_walk {
c3_o liv_o;
};

// for u3_lmdb_init() calls
static const size_t siz_i =
#if (defined(U3_CPU_aarch64) && defined(U3_OS_linux))
// 500 GiB is as large as musl on aarch64 wants to allow
0x7d00000000;
#else
0x10000000000;
#endif

#undef VERBOSE_DISK
#undef DISK_TRACE_JAM
#undef DISK_TRACE_CUE
Expand Down Expand Up @@ -680,7 +671,7 @@ u3_disk_save_meta_meta(c3_c* log_c,
{
MDB_env* dbm_u;

if ( 0 == (dbm_u = u3_lmdb_init(log_c, siz_i)) ) {
if ( 0 == (dbm_u = u3_lmdb_init(log_c, u3_Host.ops_u.siz_i)) ) {
fprintf(stderr, "disk: failed to initialize meta-lmdb\r\n");
return c3n;
}
Expand Down Expand Up @@ -1240,7 +1231,7 @@ _disk_epoc_roll(u3_disk* log_u, c3_d epo_d)
log_u->mdb_u = 0;

// initialize db of new epoch
if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) {
if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, u3_Host.ops_u.siz_i)) ) {
fprintf(stderr, "disk: failed to initialize database\r\n");
c3_free(log_u);
goto fail3;
Expand Down Expand Up @@ -1499,7 +1490,7 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
return c3n;
}

if ( 0 == (log_u->mdb_u = u3_lmdb_init(tmp_c, siz_i)) ) {
if ( 0 == (log_u->mdb_u = u3_lmdb_init(tmp_c, u3_Host.ops_u.siz_i)) ) {
fprintf(stderr, "disk: failed to initialize database at %s\r\n",
tmp_c);
return c3n;
Expand Down Expand Up @@ -1533,7 +1524,7 @@ _disk_migrate(u3_disk* log_u, c3_d eve_d)
strerror(errno));
}

if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) {
if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, u3_Host.ops_u.siz_i)) ) {
fprintf(stderr, "disk: failed to initialize database at %s\r\n",
epo_c);
return c3n;
Expand Down Expand Up @@ -1736,7 +1727,7 @@ _disk_epoc_load(u3_disk* log_u, c3_d lat_d)
snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_u->com_u->pax_c, lat_d);

// initialize latest epoch's db
if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) {
if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, u3_Host.ops_u.siz_i)) ) {
fprintf(stderr, "disk: failed to initialize database at %s\r\n",
epo_c);
return _epoc_fail;
Expand Down Expand Up @@ -1880,7 +1871,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)

if ( c3y == exs_o ) {
// load the old data.mdb file
if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) {
if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, u3_Host.ops_u.siz_i)) ) {
fprintf(stderr, "disk: failed to initialize lmdb\r\n");
c3_free(log_u);
return 0;
Expand Down
127 changes: 99 additions & 28 deletions pkg/vere/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,14 @@ _main_init(void)
u3_Host.ops_u.lut_y = 31; /* aka 2G */
u3_Host.ops_u.lom_y = 31;

u3_Host.ops_u.siz_i =
#if (defined(U3_CPU_aarch64) && defined(U3_OS_linux))
// 500 GiB is as large as musl on aarch64 wants to allow
0x7d00000000;
#else
0x10000000000;
#endif

u3C.eph_c = 0;
u3C.tos_w = 0;
}
Expand Down Expand Up @@ -297,6 +305,7 @@ _main_getopt(c3_i argc, c3_c** argv)
{ "swap-to", required_argument, NULL, 8 },
{ "toss", required_argument, NULL, 9 },
{ "behn-allow-blocked", no_argument, NULL, 10 },
{ "lmdb-map-size", required_argument, NULL, 12 },
//
{ NULL, 0, NULL, 0 },
};
Expand Down Expand Up @@ -340,6 +349,12 @@ _main_getopt(c3_i argc, c3_c** argv)
u3_Host.ops_u.beb = c3y;
break;
}
case 12: { // lmdb-map-size
if ( 1 != sscanf(optarg, "%" SCNuMAX, &u3_Host.ops_u.siz_i) ) {
return c3n;
}
break;
}
// special args
//
case c3__loom: {
Expand Down Expand Up @@ -1569,10 +1584,11 @@ _cw_info(c3_i argc, c3_c* argv[])
c3_w arg_w;

static struct option lop_u[] = {
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "lmdb-map-size", required_argument, NULL, 9 },
{ NULL, 0, NULL, 0 }
};

Expand Down Expand Up @@ -1603,6 +1619,13 @@ _cw_info(c3_i argc, c3_c* argv[])
break;
}

case 9: { // lmdb-map-size
if ( 1 != sscanf(optarg, "%" SCNuMAX, &u3_Host.ops_u.siz_i) ) {
exit(1);
}
break;
}

case '?': {
fprintf(stderr, "invalid argument\r\n");
exit(1);
Expand Down Expand Up @@ -1744,10 +1767,11 @@ _cw_cram(c3_i argc, c3_c* argv[])
c3_w arg_w;

static struct option lop_u[] = {
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "lmdb-map-size", required_argument, NULL, 9 },
{ NULL, 0, NULL, 0 }
};

Expand Down Expand Up @@ -1778,6 +1802,13 @@ _cw_cram(c3_i argc, c3_c* argv[])
break;
}

case 9: { // lmdb-map-size
if ( 1 != sscanf(optarg, "%" SCNuMAX, &u3_Host.ops_u.siz_i) ) {
exit(1);
}
break;
}

case '?': {
fprintf(stderr, "invalid argument\r\n");
exit(1);
Expand Down Expand Up @@ -1840,11 +1871,12 @@ _cw_queu(c3_i argc, c3_c* argv[])
c3_c* roc_c = 0;

static struct option lop_u[] = {
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "replay-from", required_argument, NULL, 'r' },
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "lmdb-map-size", required_argument, NULL, 9 },
{ "replay-from", required_argument, NULL, 'r' },
{ NULL, 0, NULL, 0 }
};

Expand Down Expand Up @@ -1875,6 +1907,13 @@ _cw_queu(c3_i argc, c3_c* argv[])
break;
}

case 9: { // lmdb-map-size
if ( 1 != sscanf(optarg, "%" SCNuMAX, &u3_Host.ops_u.siz_i) ) {
exit(1);
}
break;
}

case 'r': {
roc_c = strdup(optarg);
} break;
Expand Down Expand Up @@ -1948,11 +1987,12 @@ _cw_meld(c3_i argc, c3_c* argv[])
c3_w arg_w;

static struct option lop_u[] = {
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "gc-early", no_argument, NULL, 9 },
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "gc-early", no_argument, NULL, 9 },
{ "lmdb-map-size", required_argument, NULL, 10 },
{ NULL, 0, NULL, 0 }
};

Expand Down Expand Up @@ -1988,6 +2028,13 @@ _cw_meld(c3_i argc, c3_c* argv[])
break;
}

case 10: { // lmdb-map-size
if ( 1 != sscanf(optarg, "%" SCNuMAX, &u3_Host.ops_u.siz_i) ) {
exit(1);
}
break;
}

case '?': {
fprintf(stderr, "invalid argument\r\n");
exit(1);
Expand Down Expand Up @@ -2116,11 +2163,12 @@ _cw_pack(c3_i argc, c3_c* argv[])
c3_w arg_w;

static struct option lop_u[] = {
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "gc-early", no_argument, NULL, 9 },
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "gc-early", no_argument, NULL, 9 },
{ "lmdb-map-size", required_argument, NULL, 10 },
{ NULL, 0, NULL, 0 }
};

Expand Down Expand Up @@ -2156,6 +2204,13 @@ _cw_pack(c3_i argc, c3_c* argv[])
break;
}

case 10: { // lmdb-map-size
if ( 1 != sscanf(optarg, "%" SCNuMAX, &u3_Host.ops_u.siz_i) ) {
exit(1);
}
break;
}

case '?': {
fprintf(stderr, "invalid argument\r\n");
exit(1);
Expand Down Expand Up @@ -2497,10 +2552,11 @@ _cw_chop(c3_i argc, c3_c* argv[])
c3_w arg_w;

static struct option lop_u[] = {
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "loom", required_argument, NULL, c3__loom },
{ "no-demand", no_argument, NULL, 6 },
{ "swap", no_argument, NULL, 7 },
{ "swap-to", required_argument, NULL, 8 },
{ "lmdb-map-size", required_argument, NULL, 9 },
{ NULL, 0, NULL, 0 }
};

Expand Down Expand Up @@ -2531,6 +2587,13 @@ _cw_chop(c3_i argc, c3_c* argv[])
break;
}

case 9: { // lmdb-map-size
if ( 1 != sscanf(optarg, "%" SCNuMAX, &u3_Host.ops_u.siz_i) ) {
exit(1);
}
break;
}

case '?': {
fprintf(stderr, "invalid argument\r\n");
exit(1);
Expand Down Expand Up @@ -2578,14 +2641,22 @@ _cw_roll(c3_i argc, c3_c* argv[])
c3_w arg_w;

static struct option lop_u[] = {
{ "loom", required_argument, NULL, c3__loom },
{ "loom", required_argument, NULL, c3__loom },
{ "lmdb-map-size", required_argument, NULL, 6 },
{ NULL, 0, NULL, 0 }
};

u3_Host.dir_c = _main_pier_run(argv[0]);

while ( -1 != (ch_i=getopt_long(argc, argv, "", lop_u, &lid_i)) ) {
switch ( ch_i ) {
case 6: { // lmdb-map-size
if ( 1 != sscanf(optarg, "%" SCNuMAX, &u3_Host.ops_u.siz_i) ) {
exit(1);
}
break;
}

case c3__loom: {
if (_main_readw_loom("loom", &u3_Host.ops_u.lom_y)) {
exit(1);
Expand Down
1 change: 1 addition & 0 deletions pkg/vere/vere.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,7 @@
u3_even* vex_u; // --prop-*, boot enhancements

c3_o beb; // --behn-allow-blocked
c3_z siz_i; // --lmdb-map-size
} u3_opts;

/* u3_host: entire host.
Expand Down

0 comments on commit 51b2e47

Please sign in to comment.