From 607a03c243a215f179f167eb9ed3f087f2e2864c Mon Sep 17 00:00:00 2001 From: Pierre Le Marre Date: Thu, 14 Nov 2024 18:09:03 +0100 Subject: [PATCH 1/7] tools: Add --test to compile-keymap --- tools/compile-keymap.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tools/compile-keymap.c b/tools/compile-keymap.c index ebbb76c3..fd2744f4 100644 --- a/tools/compile-keymap.c +++ b/tools/compile-keymap.c @@ -49,6 +49,7 @@ static enum output_format { } output_format = FORMAT_KEYMAP; static const char *includes[64]; static size_t num_includes = 0; +static bool test = false; static void usage(char **argv) @@ -62,6 +63,8 @@ usage(char **argv) " Print this help and exit\n" " --verbose\n" " Enable verbose debugging output\n" + " --test\n" + " Test compilation but do not serialize the keymap.\n" #if ENABLE_PRIVATE_APIS " --kccgst\n" " Print a keymap which only includes the KcCGST component names instead of the full keymap\n" @@ -107,6 +110,7 @@ parse_options(int argc, char **argv, struct xkb_rule_names *names) { enum options { OPT_VERBOSE, + OPT_TEST, OPT_KCCGST, OPT_RMLVO, OPT_FROM_XKB, @@ -121,6 +125,7 @@ parse_options(int argc, char **argv, struct xkb_rule_names *names) static struct option opts[] = { {"help", no_argument, 0, 'h'}, {"verbose", no_argument, 0, OPT_VERBOSE}, + {"test", no_argument, 0, OPT_TEST}, #if ENABLE_PRIVATE_APIS {"kccgst", no_argument, 0, OPT_KCCGST}, #endif @@ -150,6 +155,9 @@ parse_options(int argc, char **argv, struct xkb_rule_names *names) case OPT_VERBOSE: verbose = true; break; + case OPT_TEST: + test = true; + break; case OPT_KCCGST: output_format = FORMAT_KCCGST; break; @@ -216,6 +224,8 @@ print_kccgst(struct xkb_context *ctx, const struct xkb_rule_names *rmlvo) if (!xkb_components_from_rules(ctx, rmlvo, &kccgst, NULL)) return false; + if (test) + goto out; printf("xkb_keymap {\n" " xkb_keycodes { include \"%s\" };\n" @@ -224,6 +234,7 @@ print_kccgst(struct xkb_context *ctx, const struct xkb_rule_names *rmlvo) " xkb_symbols { include \"%s\" };\n" "};\n", kccgst.keycodes, kccgst.types, kccgst.compat, kccgst.symbols); +out: free(kccgst.keycodes); free(kccgst.types); free(kccgst.compat); @@ -244,10 +255,14 @@ print_keymap(struct xkb_context *ctx, const struct xkb_rule_names *rmlvo) if (keymap == NULL) return false; + if (test) + goto out; + char *buf = xkb_keymap_get_as_string(keymap, XKB_KEYMAP_FORMAT_TEXT_V1); printf("%s\n", buf); free(buf); +out: xkb_keymap_unref(keymap); return true; } @@ -291,6 +306,9 @@ print_keymap_from_file(struct xkb_context *ctx) if (!keymap) { fprintf(stderr, "Couldn't create xkb keymap\n"); goto out; + } else if (test) { + success = true; + goto out; } keymap_string = xkb_keymap_get_as_string(keymap, XKB_KEYMAP_FORMAT_TEXT_V1); From 4e87cc0fab886b72c26841b393a1ece679c4bc84 Mon Sep 17 00:00:00 2001 From: Pierre Le Marre Date: Thu, 14 Nov 2024 17:41:45 +0100 Subject: [PATCH 2/7] Make context ref counting thread-safe --- src/context.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/context.c b/src/context.c index bcec16c4..74f388b0 100644 --- a/src/context.c +++ b/src/context.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "xkbcommon/xkbcommon.h" #include "utils.h" @@ -189,7 +190,7 @@ xkb_context_include_path_get(struct xkb_context *ctx, unsigned int idx) XKB_EXPORT struct xkb_context * xkb_context_ref(struct xkb_context *ctx) { - ctx->refcnt++; + atomic_fetch_add(&ctx->refcnt, 1); return ctx; } @@ -200,7 +201,7 @@ xkb_context_ref(struct xkb_context *ctx) XKB_EXPORT void xkb_context_unref(struct xkb_context *ctx) { - if (!ctx || --ctx->refcnt > 0) + if (!ctx || atomic_fetch_sub(&ctx->refcnt, 1) > 1) return; free(ctx->x11_atom_cache); From bf2a11e2f9dfe8893fd719cbec5874f41a30b75c Mon Sep 17 00:00:00 2001 From: Pierre Le Marre Date: Thu, 14 Nov 2024 17:39:33 +0100 Subject: [PATCH 3/7] Add thread-safe variant of atom table --- src/atom.c | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/atom.h | 5 +++ 2 files changed, 114 insertions(+) diff --git a/src/atom.c b/src/atom.c index d43ac381..e8632a30 100644 --- a/src/atom.c +++ b/src/atom.c @@ -80,6 +80,9 @@ #include "atom.h" #include "darray.h" #include "utils.h" +#ifdef ENABLE_KEYMAP_CACHE +#include +#endif /* FNV-1a (http://www.isthe.com/chongo/tech/comp/fnv/). */ static inline uint32_t @@ -95,6 +98,7 @@ hash_buf(const char *string, size_t len) return hash; } +#ifndef ENABLE_KEYMAP_CACHE /* * The atom table is an insert-only linear probing hash table * mapping strings to atoms. Another array maps the atoms to @@ -191,3 +195,108 @@ atom_intern(struct atom_table *table, const char *string, size_t len, bool add) assert(!"couldn't find an empty slot during probing"); } + +#else + +struct atom_table { + size_t size; + char **strings; +}; + +struct atom_table * +atom_table_new(size_t size_log2) +{ + struct atom_table *table = calloc(1, sizeof(*table)); + if (!table) + return NULL; + size_t size = 1u << size_log2; + table->size = size; + table->strings = calloc(size, sizeof(char*)); + return table; +} + +void +atom_table_free(struct atom_table *table) +{ + if (!table) + return; + + size_t count = 0; + for (size_t k = 0 ; k < table->size; k++) { + if (table->strings[k]) { + free(table->strings[k]); + count++; + } + } + free(table->strings); + free(table); +} + + +xkb_atom_t +atom_intern(struct atom_table *table, const char *string, size_t len, bool add) +{ + + uint32_t hash = hash_buf(string, len); + char *new_value = NULL; + for (size_t i = 0; i < table->size; i++) { + xkb_atom_t index_pos = (hash + i) & (table->size - 1); + if (index_pos == XKB_ATOM_NONE) + continue; + + char *existing_value = atomic_load_explicit(&table->strings[index_pos], + memory_order_acquire); + + /* Check if there is a value at the current index */ + if (!existing_value) { + /* No value defined. Check if we are allowed to add one */ + if (!add) + return XKB_ATOM_NONE; + + /* + * Prepare addition: duplicate string. + * Warning: This may not be our first attempt! + */ + if (!new_value) + new_value = strndup(string, len); + if (!new_value) { + /* Failed memory allocation */ + // FIXME: error handling? + return XKB_ATOM_NONE; + } + /* Try to register a new entry */ + if (atomic_compare_exchange_strong_explicit( + &table->strings[index_pos], &existing_value, new_value, + memory_order_release, memory_order_acquire)) { + return index_pos; + } + /* + * We were not fast enough to take the spot. + * But maybe the newly added value is our value, so read it again. + */ + existing_value = atomic_load_explicit(&table->strings[index_pos], + memory_order_acquire); + } + + /* Check the value are equal */ + if (strncmp(existing_value, string, len) == 0 && + existing_value[len] == '\0') { + /* We may have tried unsuccessfully to add the string */ + free(new_value); + return index_pos; + } + /* Hash collision: try next atom */ + } + + free(new_value); + assert(!"couldn't find an empty slot during probing"); +} + +const char * +atom_text(struct atom_table *table, xkb_atom_t atom) +{ + assert(atom < table->size); + return table->strings[atom]; +} + +#endif diff --git a/src/atom.h b/src/atom.h index 49478db9..072ac0db 100644 --- a/src/atom.h +++ b/src/atom.h @@ -30,8 +30,13 @@ typedef uint32_t xkb_atom_t; struct atom_table; +#ifdef ENABLE_KEYMAP_CACHE +struct atom_table * +atom_table_new(size_t size); +#else struct atom_table * atom_table_new(void); +#endif void atom_table_free(struct atom_table *table); From fb9a4f459dd9f1b352ede867e6f9527966540bee Mon Sep 17 00:00:00 2001 From: Pierre Le Marre Date: Thu, 14 Nov 2024 17:33:28 +0100 Subject: [PATCH 4/7] xkbcomp: Add cloning utils --- src/xkbcomp/ast-build.c | 397 +++++++++++++++++++++++++++++++++++++ src/xkbcomp/ast-build.h | 3 + src/xkbcomp/ast.h | 17 ++ src/xkbcomp/xkbcomp-priv.h | 3 + 4 files changed, 420 insertions(+) diff --git a/src/xkbcomp/ast-build.c b/src/xkbcomp/ast-build.c index 1d2ae4a7..24304dec 100644 --- a/src/xkbcomp/ast-build.c +++ b/src/xkbcomp/ast-build.c @@ -695,6 +695,160 @@ FreeExpr(ExprDef *expr) } } +static ExprDef * +DupValueExpr(ExprDef *expr) +{ + if (!expr) + return NULL; + switch(expr->expr.value_type) { + case EXPR_TYPE_BOOLEAN: + return memdup(expr, 1, sizeof(ExprBoolean)); + case EXPR_TYPE_INT: + return memdup(expr, 1, sizeof(ExprInteger)); + case EXPR_TYPE_FLOAT: + return memdup(expr, 1, sizeof(ExprFloat)); + case EXPR_TYPE_STRING: + return memdup(expr, 1, sizeof(ExprString)); + case EXPR_TYPE_KEYNAME: + return memdup(expr, 1, sizeof(ExprKeyName)); + default: + return memdup(expr, 1, sizeof(ExprCommon)); + } +} + +static ExprDef * +DupExpr(ExprDef *expr) +{ + if (!expr) + return NULL; + + ExprDef** action; + ExprDef *new; + + switch (expr->expr.op) { + case EXPR_VALUE: + new = DupValueExpr(expr); + if (!new) + return NULL; + break; + + case EXPR_IDENT: + new = memdup(expr, 1, sizeof(ExprIdent)); + if (!new) + return NULL; + break; + + case EXPR_NEGATE: + case EXPR_UNARY_PLUS: + case EXPR_NOT: + case EXPR_INVERT: + new = memdup(expr, 1, sizeof(ExprUnary)); + if (!new) + return NULL; + if (expr->unary.child) { + new->unary.child = (ExprDef *) DupStmt((ParseCommon *) expr->unary.child); + if (!new->unary.child) + goto error; + } + break; + + case EXPR_DIVIDE: + case EXPR_ADD: + case EXPR_SUBTRACT: + case EXPR_MULTIPLY: + case EXPR_ASSIGN: + new = memdup(expr, 1, sizeof(ExprBinary)); + if (!new) + return NULL; + new->binary.left = (ExprDef *) DupStmt((ParseCommon *) expr->binary.left); + new->binary.right = (ExprDef *) DupStmt((ParseCommon *) expr->binary.right); + if ((expr->binary.left && !new->binary.left) || + (expr->binary.right && !new->binary.right)) + goto error; + break; + + case EXPR_ACTION_DECL: + new = memdup(expr, 1, sizeof(ExprAction)); + if (!new) + return NULL; + if (expr->action.args) { + new->action.args = (ExprDef *) DupStmt((ParseCommon *) expr->action.args); + if (!new->action.args) + goto error; + } + break; + + case EXPR_ACTION_LIST: + new = memdup(expr, 1, sizeof(ExprActionList)); + if (!new) + return NULL; + do { + bool ok = true; + darray_init(new->actions.actions); + darray_init(new->actions.actionsMapIndex); + darray_init(new->actions.actionsNumEntries); + darray_foreach(action, expr->actions.actions) { + ExprDef* new_action = (ExprDef *) DupStmt((ParseCommon *) *action); + if (!new_action) { + ok = false; + break; + } + darray_append(new->actions.actions, new_action); + } + if (!ok) { + darray_foreach(action, new->actions.actions) { + FreeStmt((ParseCommon *) *action); + } + darray_free(new->actions.actions); + goto error; + } + darray_copy(new->actions.actionsMapIndex, expr->actions.actionsMapIndex); + darray_copy(new->actions.actionsNumEntries, expr->actions.actionsNumEntries); + } while (0); + break; + + case EXPR_FIELD_REF: + new = memdup(expr, 1, sizeof(ExprFieldRef)); + if (!new) + return NULL; + break; + + case EXPR_ARRAY_REF: + new = memdup(expr, 1, sizeof(ExprArrayRef)); + if (!new) + return NULL; + if (expr->array_ref.entry) { + new->array_ref.entry = (ExprDef *) DupStmt((ParseCommon *) expr->array_ref.entry); + if (!new->array_ref.entry) + goto error; + } + break; + + case EXPR_KEYSYM_LIST: + new = memdup(expr, 1, sizeof(ExprKeysymList)); + if (!new) + return NULL; + darray_init(new->keysym_list.syms); + darray_init(new->keysym_list.symsMapIndex); + darray_init(new->keysym_list.symsNumEntries); + darray_copy(new->keysym_list.syms, expr->keysym_list.syms); + darray_copy(new->keysym_list.symsMapIndex, expr->keysym_list.symsMapIndex); + darray_copy(new->keysym_list.symsNumEntries, expr->keysym_list.symsNumEntries); + break; + + default: + new = memdup(expr, 1, sizeof(ExprDef)); + if (!new) + return NULL; + break; + } + return new; +error: + FreeExpr(new); + free(new); + return NULL; +} + static void FreeInclude(IncludeStmt *incl) { @@ -714,6 +868,36 @@ FreeInclude(IncludeStmt *incl) } } +static IncludeStmt * +DupInclude(IncludeStmt *incl) +{ + if (!incl) + return NULL; + IncludeStmt *new = memdup(incl, 1, sizeof(*incl)); + IncludeStmt *next = new; + while (1) { + if (!next) + goto error; + next->stmt = strdup_safe(incl->stmt); + next->file = strdup_safe(incl->file); + next->map = strdup_safe(incl->map); + next->modifier = strdup_safe(incl->modifier); + if ((incl->stmt && !next->stmt) || (incl->file && !next->file) || + (incl->map && !next->map) || (incl->modifier && !next->modifier)) { + goto error; + } + incl = incl->next_incl; + if (!incl) + break; + next->next_incl = memdup(incl, 1, sizeof(*incl)); + next = next->next_incl; + } + return new; +error: + FreeInclude(new); + return NULL; +} + void FreeStmt(ParseCommon *stmt) { @@ -770,6 +954,164 @@ FreeStmt(ParseCommon *stmt) } } +ParseCommon * +DupStmt(ParseCommon *stmt) +{ + if (!stmt) + return NULL; + ParseCommon new = {0}; + ParseCommon *next = &new; + do { + // FIXME debug + // fprintf(stderr, "~~~ CopyStmt: type=%d\n", stmt->type); + + switch (stmt->type) { + case STMT_INCLUDE: + next->next = (ParseCommon *) DupInclude((IncludeStmt *) stmt); + if (!next->next) + goto error; + break; + case STMT_KEYCODE: + do { + KeycodeDef *def = memdup(stmt, 1, sizeof(KeycodeDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + } while (0); + break; + case STMT_ALIAS: + do { + KeyAliasDef *def = memdup(stmt, 1, sizeof(KeyAliasDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + } while (0); + break; + case STMT_EXPR: + next->next = (ParseCommon *) DupExpr((ExprDef *) stmt); + if (!next->next) + goto error; + break; + case STMT_VAR: + do { + VarDef *def = memdup(stmt, 1, sizeof(VarDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + def->name = (ExprDef *) DupStmt((ParseCommon *) ((VarDef *)stmt)->name); + def->value = (ExprDef *) DupStmt((ParseCommon *) ((VarDef *)stmt)->value); + if ((((VarDef *)stmt)->name && !def->name) || + (((VarDef *)stmt)->value && !def->value)) + goto error; + } while (0); + break; + case STMT_TYPE: + do { + KeyTypeDef *def = memdup(stmt, 1, sizeof(KeyTypeDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + def->body = (VarDef*) DupStmt((ParseCommon *) ((KeyTypeDef *) stmt)->body); + if (((KeyTypeDef *) stmt)->body && !def->body) + goto error; + } while (0); + break; + case STMT_INTERP: + do { + InterpDef *def = memdup(stmt, 1, sizeof(InterpDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + def->match = (ExprDef *) DupStmt((ParseCommon *) ((InterpDef *) stmt)->match); + def->def = (VarDef*) DupStmt((ParseCommon *) ((InterpDef *) stmt)->def); + if ((((InterpDef *) stmt)->match && !def->match) || + (((InterpDef *) stmt)->def && !def->def)) + goto error; + } while (0); + break; + case STMT_VMOD: + do { + VModDef *def = memdup(stmt, 1, sizeof(VModDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + def->value = (ExprDef *) DupStmt((ParseCommon *) ((VModDef *) stmt)->value); + if (((VModDef *) stmt)->value && !def->value) + goto error; + } while (0); + break; + case STMT_SYMBOLS: + do { + SymbolsDef *def = memdup(stmt, 1, sizeof(SymbolsDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + def->symbols = (VarDef *) DupStmt((ParseCommon *) ((SymbolsDef *) stmt)->symbols); + if (((SymbolsDef *) stmt)->symbols && !def->symbols) + goto error; + } while (0); + break; + case STMT_MODMAP: + do { + ModMapDef *def = memdup(stmt, 1, sizeof(ModMapDef)); + if (!def) + goto error; + def->keys = (ExprDef *) DupStmt((ParseCommon *) ((ModMapDef *) stmt)->keys); + if (((ModMapDef *) stmt)->keys && !def->keys) + goto error; + next->next = (ParseCommon *)def; + } while (0); + break; + case STMT_GROUP_COMPAT: + do { + GroupCompatDef *def = memdup(stmt, 1, sizeof(GroupCompatDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + def->def = (ExprDef *) DupStmt((ParseCommon *) ((GroupCompatDef *) stmt)->def); + if (((GroupCompatDef *) stmt)->def && !def->def) + goto error; + } while (0); + break; + case STMT_LED_MAP: + do { + LedMapDef *def = memdup(stmt, 1, sizeof(LedMapDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + def->body = (VarDef *) DupStmt((ParseCommon *) ((LedMapDef *) stmt)->body); + if (((LedMapDef *) stmt)->body && !def->body) + goto error; + } while (0); + break; + case STMT_LED_NAME: + do { + LedNameDef *def = memdup(stmt, 1, sizeof(LedNameDef)); + if (!def) + goto error; + next->next = (ParseCommon *)def; + def->name = (ExprDef *) DupStmt((ParseCommon *) ((LedNameDef *) stmt)->name); + if (((LedNameDef *) stmt)->name && !def->name) + goto error; + } while (0); + break; + default: + break; + } + + stmt = stmt->next; + if (!stmt) + break; + next = next->next; + if (!next) + goto error; + } while (1); + return new.next; +error: + FreeStmt(new.next); + return NULL; +} + void FreeXkbFile(XkbFile *file) { @@ -802,6 +1144,61 @@ FreeXkbFile(XkbFile *file) } } +XkbFile * +DupXkbFile(XkbFile *file) +{ + if (!file) + return NULL; + XkbFile *new = memdup(file, 1, sizeof(*file)); + XkbFile *next = new; + // FIXME debug + // fprintf(stderr, "~~~ DupXkbFile: start\n"); + do { + // FIXME debug + // fprintf(stderr, "~~~ DupXkbFile: file_type=%d, %s\n", file->file_type, file->name); + if (!next) + goto error; + next->common.next = NULL; + next->defs = NULL; + if (file->name) { + next->name = strdup(file->name); + if (!next->name) + goto error; + } + switch (file->file_type) { + case FILE_TYPE_KEYMAP: + next->defs = (ParseCommon *) DupXkbFile((XkbFile *) file->defs); + if (!next->defs) + goto error; + break; + + case FILE_TYPE_TYPES: + case FILE_TYPE_COMPAT: + case FILE_TYPE_SYMBOLS: + case FILE_TYPE_KEYCODES: + case FILE_TYPE_GEOMETRY: + // FIXME debug + // fprintf(stderr, "~~~ DupXkbFile: stmt=%d\n", file->defs->type); + next->defs = DupStmt(file->defs); + if (!next->defs) + goto error; + break; + + default: + break; + } + file = (XkbFile *) file->common.next; + if (!file) + break; + next->common.next = memdup(file, 1, sizeof(*file)); + next = (XkbFile *) next->common.next; + } while(1); + return new; +error: + FreeXkbFile(new); + return NULL; +} + static const char *xkb_file_type_strings[_FILE_TYPE_NUM_ENTRIES] = { [FILE_TYPE_KEYCODES] = "xkb_keycodes", [FILE_TYPE_TYPES] = "xkb_types", diff --git a/src/xkbcomp/ast-build.h b/src/xkbcomp/ast-build.h index 8f0c17ee..b94766ff 100644 --- a/src/xkbcomp/ast-build.h +++ b/src/xkbcomp/ast-build.h @@ -131,4 +131,7 @@ XkbFileCreate(enum xkb_file_type type, char *name, ParseCommon *defs, void FreeStmt(ParseCommon *stmt); +ParseCommon * +DupStmt(ParseCommon *stmt); + #endif diff --git a/src/xkbcomp/ast.h b/src/xkbcomp/ast.h index 453d3831..f006677f 100644 --- a/src/xkbcomp/ast.h +++ b/src/xkbcomp/ast.h @@ -346,6 +346,23 @@ typedef struct { VarDef *body; } LedMapDef; +union Statement { + ParseCommon _pc; + IncludeStmt _incl; + ExprDef _e; + VarDef _v; + KeycodeDef _k; + KeyAliasDef _a; + KeyTypeDef _kt; + InterpDef _interp; + VModDef _vm; + SymbolsDef _s; + ModMapDef _mm; + GroupCompatDef _gc; + LedMapDef _lm; + LedNameDef _ln; +}; + enum xkb_map_flags { MAP_IS_DEFAULT = (1 << 0), MAP_IS_PARTIAL = (1 << 1), diff --git a/src/xkbcomp/xkbcomp-priv.h b/src/xkbcomp/xkbcomp-priv.h index 9156922c..50ae29a1 100644 --- a/src/xkbcomp/xkbcomp-priv.h +++ b/src/xkbcomp/xkbcomp-priv.h @@ -52,6 +52,9 @@ XkbParseString(struct xkb_context *ctx, void FreeXkbFile(XkbFile *file); +XkbFile * +DupXkbFile(XkbFile *file); + XkbFile * XkbFileFromComponents(struct xkb_context *ctx, const struct xkb_component_names *kkctgs); From e86e58ff87b9b6e03f26ddf2cc24b5e4cc37858e Mon Sep 17 00:00:00 2001 From: Pierre Le Marre Date: Thu, 14 Nov 2024 17:46:28 +0100 Subject: [PATCH 5/7] xkbcomp: Propose a variant using a cache for keymap components --- src/context.c | 5 ++ src/context.h | 9 ++++ src/xkbcomp/cache.c | 104 ++++++++++++++++++++++++++++++++++++++++++ src/xkbcomp/cache.h | 59 ++++++++++++++++++++++++ src/xkbcomp/include.c | 22 ++++++++- 5 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 src/xkbcomp/cache.c create mode 100644 src/xkbcomp/cache.h diff --git a/src/context.c b/src/context.c index 74f388b0..c179b3e9 100644 --- a/src/context.c +++ b/src/context.c @@ -313,7 +313,12 @@ xkb_context_new(enum xkb_context_flags flags) return NULL; } +#ifdef ENABLE_KEYMAP_CACHE + /* NOTE: Size should be adusted to deal with xkeyboard-config database */ + ctx->atom_table = atom_table_new(14); +#else ctx->atom_table = atom_table_new(); +#endif if (!ctx->atom_table) { xkb_context_unref(ctx); return NULL; diff --git a/src/context.h b/src/context.h index 3b513da5..51c9db32 100644 --- a/src/context.h +++ b/src/context.h @@ -26,8 +26,13 @@ #ifndef CONTEXT_H #define CONTEXT_H +#include "xkbcommon/xkbcommon.h" #include "atom.h" #include "messages-codes.h" +#include "src/utils.h" +#ifdef ENABLE_KEYMAP_CACHE +#include "xkbcomp/cache.h" +#endif struct xkb_context { int refcnt; @@ -55,6 +60,10 @@ struct xkb_context { unsigned int use_environment_names : 1; unsigned int use_secure_getenv : 1; + +#ifdef ENABLE_KEYMAP_CACHE + struct xkb_keymap_cache *keymap_cache; +#endif }; char * diff --git a/src/xkbcomp/cache.c b/src/xkbcomp/cache.c new file mode 100644 index 00000000..edb3046e --- /dev/null +++ b/src/xkbcomp/cache.c @@ -0,0 +1,104 @@ +/* + * Copyright © 2024 Pierre Le Marre + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "config.h" + +#include +#include + +#include "cache.h" +#include "xkbcomp-priv.h" + +static pthread_mutex_t keymap_cache_mutext = PTHREAD_MUTEX_INITIALIZER; + +struct xkb_keymap_cache* +xkb_keymap_cache_new(void) +{ + struct xkb_keymap_cache* cache = calloc(1, sizeof(struct xkb_keymap_cache)); + if (!cache) + return NULL; + if (!hcreate_r(XKB_KEYMAP_CACHE_SIZE, &cache->index)) + return NULL; + darray_init(cache->data); + return cache; +} + +void +xkb_keymap_cache_free(struct xkb_keymap_cache *cache) +{ + if (!cache) + return; + hdestroy_r(&cache->index); + struct xkb_keymap_cache_entry* entry; + darray_foreach(entry, cache->data) { + free(entry->key); + FreeXkbFile(entry->data); + } + darray_free(cache->data); + free(cache); +} + +bool +xkb_keymap_cache_add(struct xkb_keymap_cache *cache, char *key, XkbFile *data) +{ + struct xkb_keymap_cache_entry entry = { + .key = strdup(key), + .data = DupXkbFile(data) + }; + if (!entry.key || !entry.data) + goto error; + size_t idx = darray_size(cache->data); + pthread_mutex_lock(&keymap_cache_mutext); + darray_append(cache->data, entry); + if (darray_size(cache->data) == idx) { + pthread_mutex_unlock(&keymap_cache_mutext); + goto error; + } + ENTRY input = { + .key = entry.key, + .data = entry.data + }; + ENTRY *output; + bool ret = !hsearch_r(input, ENTER, &output, &cache->index); + pthread_mutex_unlock(&keymap_cache_mutext); + return ret; +error: + if (entry.key) + free(entry.key); + if (entry.data) + FreeXkbFile(entry.data); + return false; +} + +XkbFile* +xkb_keymap_cache_search(struct xkb_keymap_cache *cache, char *key) +{ + ENTRY input = { + .key = key, + .data = NULL + }; + ENTRY *output; + if (!hsearch_r(input, FIND, &output, &cache->index)) + return NULL; + return DupXkbFile(output->data); +} diff --git a/src/xkbcomp/cache.h b/src/xkbcomp/cache.h new file mode 100644 index 00000000..707ac0b6 --- /dev/null +++ b/src/xkbcomp/cache.h @@ -0,0 +1,59 @@ +/* + * Copyright © 2024 Pierre Le Marre + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef XKBCOMP_CACHE_H +#define XKBCOMP_CACHE_H + +#include +#include + +#include "xkbcommon/xkbcommon.h" +#include "src/darray.h" +#include "src/atom.h" +#include "ast.h" + +#define XKB_KEYMAP_CACHE_SIZE 100000 + +struct xkb_keymap_cache_entry { + char *key; + XkbFile *data; +}; + +struct xkb_keymap_cache { + struct hsearch_data index; + darray(struct xkb_keymap_cache_entry) data; +}; + +struct xkb_keymap_cache* +xkb_keymap_cache_new(void); + +void +xkb_keymap_cache_free(struct xkb_keymap_cache *cache); + +bool +xkb_keymap_cache_add(struct xkb_keymap_cache *cache, char *key, XkbFile *data); + +XkbFile* +xkb_keymap_cache_search(struct xkb_keymap_cache *cache, char *key); + +#endif diff --git a/src/xkbcomp/include.c b/src/xkbcomp/include.c index 620b2f4f..f893ac27 100644 --- a/src/xkbcomp/include.c +++ b/src/xkbcomp/include.c @@ -56,6 +56,10 @@ #include "xkbcomp-priv.h" #include "include.h" +#ifdef ENABLE_KEYMAP_CACHE +#include "cache.h" +#endif + /** * Parse an include statement. Each call returns a file name, along with * (possibly) a specific map in the file, an explicit group designator, and @@ -301,6 +305,18 @@ ProcessIncludeFile(struct xkb_context *ctx, IncludeStmt *stmt, XkbFile *xkb_file = NULL; unsigned int offset = 0; +#ifdef ENABLE_KEYMAP_CACHE + char key[1024]; + if (ctx->keymap_cache) { + int ret = snprintf(key, ARRAY_SIZE(key), "%s/%s\n%s", + DirectoryForInclude(file_type), stmt->file, + (stmt->map) ? stmt->map : ""); + if (ret > 0 && + (xkb_file = xkb_keymap_cache_search(ctx->keymap_cache, key))) { + return xkb_file; + } + } +#endif file = FindFileInXkbPath(ctx, stmt->file, file_type, NULL, &offset); if (!file) return NULL; @@ -337,6 +353,10 @@ ProcessIncludeFile(struct xkb_context *ctx, IncludeStmt *stmt, "Couldn't process include statement for '%s'\n", stmt->file); } - +#ifdef ENABLE_KEYMAP_CACHE + if (ctx->keymap_cache) { + xkb_keymap_cache_add(ctx->keymap_cache, key, xkb_file); + } +#endif return xkb_file; } From a4d2def6b4b5bcd35d1f4c4a173482a3ece1424d Mon Sep 17 00:00:00 2001 From: Pierre Le Marre Date: Thu, 14 Nov 2024 17:59:40 +0100 Subject: [PATCH 6/7] tools: Keymap compiler server --- meson.build | 13 + test/xkeyboard-config-test.py.in | 358 +++++++++++++++++---- tools/compile-keymap-server.c | 516 +++++++++++++++++++++++++++++++ 3 files changed, 835 insertions(+), 52 deletions(-) create mode 100644 tools/compile-keymap-server.c diff --git a/meson.build b/meson.build index 49509bce..970d275e 100644 --- a/meson.build +++ b/meson.build @@ -512,6 +512,19 @@ if build_tools c_args: ['-DENABLE_PRIVATE_APIS'], include_directories: [include_directories('src', 'include')], install: false) + thread_dep = dependency('threads', required: false) + if thread_dep.found() and cc.has_header('sys/un.h') + # The same tool again, but for batch processing. + executable('compile-keymap-server', + 'tools/compile-keymap-server.c', + 'src/xkbcomp/cache.c', + 'src/xkbcomp/cache.h', + libxkbcommon_sources, + dependencies: [tools_dep, thread_dep], + c_args: ['-DENABLE_KEYMAP_SOCKET', '-DENABLE_KEYMAP_CACHE'], + include_directories: [include_directories('src', 'include')], + install: false) + endif # Tool: compose executable('xkbcli-compile-compose', diff --git a/test/xkeyboard-config-test.py.in b/test/xkeyboard-config-test.py.in index a4aa57e1..854c38cc 100755 --- a/test/xkeyboard-config-test.py.in +++ b/test/xkeyboard-config-test.py.in @@ -3,17 +3,25 @@ from __future__ import annotations import argparse +import ctypes +import gzip import itertools import multiprocessing import os +import socket import subprocess import sys +import tempfile +from time import sleep +import traceback import xml.etree.ElementTree as ET from abc import ABCMeta, abstractmethod from dataclasses import dataclass +from functools import partial from pathlib import Path from typing import ( TYPE_CHECKING, + BinaryIO, ClassVar, Iterable, Iterator, @@ -53,7 +61,6 @@ class RMLVO: variant: str | None option: str | None - @property def __iter__(self) -> Iterator[str | None]: yield self.rules yield self.model @@ -95,7 +102,7 @@ class RMLVO: class Invocation(RMLVO, metaclass=ABCMeta): exitstatus: int = 77 # default to “skipped” error: str | None = None - keymap: str = "" + keymap: bytes = b"" command: str = "" # The fully compiled keymap def __str_iter(self) -> Iterator[str]: @@ -124,12 +131,42 @@ class Invocation(RMLVO, metaclass=ABCMeta): def escape(s: str) -> str: return s.replace('"', '\\"') - @abstractmethod - def _run(self) -> Self: ... + def _write(self, fd: BinaryIO) -> None: + fd.write(f"// {self.to_yaml(self.rmlvo)}\n".encode("utf-8")) + fd.write(self.keymap) + + def _write_keymap(self, output_dir: Path, compress: int) -> None: + layout = self.layout + if self.variant: + layout += f"({self.variant})" + (output_dir / self.model).mkdir(exist_ok=True) + keymap_file = output_dir / self.model / layout + if compress: + keymap_file = keymap_file.with_suffix(".gz") + with gzip.open(keymap_file, "wb", compresslevel=compress) as fd: + self._write(fd) + fd.close() + else: + with keymap_file.open("wb") as fd: + self._write(fd) + + def _print_result(self, short: bool, verbose: bool) -> None: + if self.exitstatus != 0: + target = sys.stderr + else: + target = sys.stdout if verbose else None + + if target: + if short: + print("-", self.to_yaml(self.short), file=target) + else: + print(self, file=target) @classmethod - def run(cls, x: Self) -> Self: - return x._run() + @abstractmethod + def run( + cls, i: Self, output_dir: Path | None, compress: int, *args, **kwargs + ) -> Self: ... @classmethod def run_all( @@ -141,6 +178,9 @@ class Invocation(RMLVO, metaclass=ABCMeta): verbose: bool, short: bool, progress_bar: ProgressBar[Iterable[Self]], + chunksize: int, + compress: int, + **kwargs, ) -> bool: if keymap_output_dir: try: @@ -149,45 +189,21 @@ class Invocation(RMLVO, metaclass=ABCMeta): print(e, file=sys.stderr) return False - keymap_file: Path | None = None - keymap_file_fd: TextIO | None = None - failed = False with multiprocessing.Pool(njobs) as p: - results = p.imap_unordered(cls.run, combos) + f = partial( + cls.run, + output_dir=keymap_output_dir, + compress=compress, + ) + results = p.imap_unordered(f, combos, chunksize=chunksize) for invocation in progress_bar( results, total=combos_count, file=sys.stdout ): if invocation.exitstatus != 0: failed = True - target = sys.stderr - else: - target = sys.stdout if verbose else None - - if target: - if short: - print("-", cls.to_yaml(invocation.short), file=target) - else: - print(invocation, file=target) - - if keymap_output_dir: - # we're running through the layouts in a somewhat sorted manner, - # so let's keep the fd open until we switch layouts - layout = invocation.layout - if invocation.variant: - layout += f"({invocation.variant})" - (keymap_output_dir / invocation.model).mkdir(exist_ok=True) - fname = keymap_output_dir / invocation.model / layout - if fname != keymap_file: - keymap_file = fname - if keymap_file_fd: - keymap_file_fd.close() - keymap_file_fd = open(keymap_file, "a") - - print(f"// {cls.to_yaml(invocation.rmlvo)}", file=keymap_file_fd) - print(invocation.keymap, file=keymap_file_fd) - assert keymap_file_fd - keymap_file_fd.flush() + + invocation._print_result(short, verbose) return failed @@ -196,7 +212,19 @@ class Invocation(RMLVO, metaclass=ABCMeta): class XkbCompInvocation(Invocation): xkbcomp_args: ClassVar[tuple[str, ...]] = ("xkbcomp", "-xkb", "-", "-") - def _run(self) -> Self: + @classmethod + def run( + cls, + i: Self, + output_dir: Path | None, + compress: int, + *args, + **kwargs, + ) -> Self: + i._run(output_dir, compress) + return i + + def _run(self, output_dir: Path | None, compress: int) -> None: args = ( "setxkbmap", "-print", @@ -228,39 +256,252 @@ class XkbCompInvocation(Invocation): self.error = "failed to compile keymap" self.exitstatus = xkbcomp.returncode else: - self.keymap = stdout + self.keymap = stdout.encode("utf-8") self.exitstatus = 0 - return self + + if output_dir: + self._write_keymap(output_dir, compress) @dataclass class XkbcommonInvocation(Invocation): UNRECOGNIZED_KEYSYM_ERROR: ClassVar[str] = "XKB-107" - def _run(self) -> Self: + def _check_stderr(self, stderr: str) -> bool: + if self.UNRECOGNIZED_KEYSYM_ERROR in stderr: + for line in stderr.splitlines(): + if self.UNRECOGNIZED_KEYSYM_ERROR in line: + self.error = line + break + self.exitstatus = 99 # tool doesn't generate this one + return False + else: + self.exitstatus = 0 + return True + + @classmethod + def run( + cls, + i: Self, + output_dir: Path | None, + compress: int, + *args, + **kwargs, + ) -> Self: + i._run(output_dir, compress) + return i + + def _run(self, output_dir: Path | None, compress: int) -> None: args = ( "xkbcli-compile-keymap", # this is run in the builddir # Not needed, because we set XKB_LOG_LEVEL and XKB_LOG_VERBOSITY in env # "--verbose", *itertools.chain.from_iterable((f"--{k}", v) for k, v in self.rmlvo), ) + if not output_dir: + args += ("--test",) self.command = " ".join(args) try: completed = subprocess.run(args, text=True, check=True, capture_output=True) - if self.UNRECOGNIZED_KEYSYM_ERROR in completed.stderr: - for line in completed.stderr.splitlines(): - if self.UNRECOGNIZED_KEYSYM_ERROR in line: - self.error = line - self.exitstatus = 99 # tool doesn't generate this one - break - else: - self.exitstatus = 0 - self.keymap = completed.stdout except subprocess.CalledProcessError as err: self.error = "failed to compile keymap" self.exitstatus = err.returncode + else: + if self._check_stderr(completed.stderr): + self.keymap = completed.stdout.encode("utf-8") + if output_dir: + self._write_keymap(output_dir, compress) + + +@dataclass +class CompileKeymapPool: + count: int + pool: tuple[subprocess.Popen] = () + tmp: tempfile.TemporaryDirectory | None = None + socket: Path | None = None + + @property + def sockets(self) -> Iterable[Path]: + if self.socket: + yield self.socket + elif self.tmp: + for k in range(self.count): + yield Path(self.tmp.name) / str(k) + + def __enter__(self): + if not self.pool and not self.socket: + self.tmp = tempfile.TemporaryDirectory() + if self.pool: + for p in self.pool: + p.kill() + pool: list[subprocess.Popen] = [] + for path in self.sockets: + args = ("compile-keymap-server", "--socket", str(path)) + pool.append( + subprocess.Popen( + args, + close_fds=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + # stderr=None, + ) + ) + self.pool = tuple(pool) return self + def __exit__(self, exc_type, exc_value, traceback): + if self.socket: + return + for path in self.sockets: + self.bye(path) + sleep(0.5) + if self.pool: + for p in self.pool: + p.kill() + self.pool = () + self.tmp.cleanup() + + @classmethod + def _get_response(cls, s: socket.socket, more: bool) -> tuple[bool, int, bytes]: + # Get size of the message + response = s.recv(ctypes.sizeof(ctypes.c_ssize_t)) + response_size = int.from_bytes(response, byteorder=sys.byteorder, signed=True) + # Get message + response = b"" + if response_size > 0: + to_read = response_size + ba = bytearray(response_size) + view = memoryview(ba) + while to_read: + nbytes = s.recv_into(view, to_read, 0) + view = view[nbytes:] + to_read -= nbytes + response = bytes(ba) + assert len(response) == response_size, (response_size, len(response)) + ok = True + else: + response = b"" + ok = not (response_size < 0) + # Confirm reception + s.sendall(b"1" if more else b"0") + return ok, response_size, response + + @classmethod + def message(cls, socket_path: Path, msg: bytes) -> tuple[bool, bytes, bytes]: + response = b"" + response_size = 0 + stderr = b"" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + ok = True + try: + s.settimeout(6) + s.connect(str(socket_path)) + s.sendall(msg) + _ok, response_size, response = cls._get_response(s, True) + ok &= _ok + _ok, response_size, stderr = cls._get_response(s, False) + ok &= _ok + except OSError as err: + print( + "Socket error:", err, ok, response_size, response, file=sys.stderr + ) + traceback.print_exception(err, file=sys.stderr) + ok = False + s.shutdown(socket.SHUT_RDWR) + return ok, response, stderr + + @classmethod + def bye(cls, socket_path: Path) -> None: + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.settimeout(6) + s.connect(str(socket_path)) + s.sendall(b"\x1b") + s.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + @classmethod + def get_keymap( + cls, socket_path: Path, rmlvo: RMLVO, serialize: bool + ) -> tuple[bool, bytes, bytes]: + query = f"{int(serialize)}\n" + "\n".join(v or "" for v in rmlvo) + return cls.message(socket_path, query.encode()) + + +@dataclass +class XkbcommonInvocationServer(XkbcommonInvocation): + def _write(self, fd: BinaryIO) -> None: + super()._write(fd) + if self.keymap: + fd.write(b"\n") + + @classmethod + def run( + cls, + i: Self, + output_dir: Path | None, + compress: int, + socket_path: Path, + *args, + **kwargs, + ) -> Self: + i._run(output_dir, compress, socket_path) + return i + + def _run(self, output_dir: Path | None, compress: int, socket_path: Path) -> None: + serialize = bool(output_dir) + ok, self.keymap, stderr = CompileKeymapPool.get_keymap( + socket_path, self, serialize + ) + + self.exitstatus = 0 if ok else 1 + self._check_stderr(stderr.decode("utf-8")) + + if output_dir: + self._write_keymap(output_dir, compress) + + @classmethod + def run_all( + cls, + combos: Iterable[Self], + combos_count: int, + njobs: int, + keymap_output_dir: Path | None, + verbose: bool, + short: bool, + progress_bar: ProgressBar[Iterable[Self]], + chunksize: int, + compress: int, + **kwargs, + ) -> bool: + if keymap_output_dir: + try: + keymap_output_dir.mkdir(parents=True) + except FileExistsError as e: + print(e, file=sys.stderr) + return False + + failed = False + + with CompileKeymapPool(1, socket=kwargs.get("server_socket")) as kmp: + with multiprocessing.Pool(njobs, maxtasksperchild=1000) as p: + f = partial( + cls.run, + socket_path=tuple(kmp.sockets)[0], + output_dir=keymap_output_dir, + compress=compress, + ) + results = p.imap_unordered(f, combos, chunksize=chunksize) + for invocation in progress_bar( + results, total=combos_count, file=sys.stdout + ): + if invocation.exitstatus != 0: + failed = True + invocation._print_result(short, verbose) + + return failed + @dataclass class Layout: @@ -411,6 +652,7 @@ def main() -> NoReturn: ) tools: dict[str, type[Invocation]] = { "libxkbcommon": XkbcommonInvocation, + "libxkbcommon-server": XkbcommonInvocationServer, "xkbcomp": XkbCompInvocation, } parser.add_argument( @@ -437,12 +679,14 @@ def main() -> NoReturn: type=Path, help="Directory to print compiled keymaps to", ) + parser.add_argument("--compress", type=int, default=0) parser.add_argument( "--model", default="", type=str, help="Only test the given model" ) parser.add_argument( "--layout", default=WILDCARD, type=str, help="Only test the given layout" ) + parser.add_argument("--chunksize", default=1, type=int) parser.add_argument( "--variant", default=WILDCARD, @@ -455,6 +699,7 @@ def main() -> NoReturn: parser.add_argument( "--no-iterations", "-1", action="store_true", help="Only test one combo" ) + parser.add_argument("--socket", type=Path) args = parser.parse_args() @@ -484,7 +729,16 @@ def main() -> NoReturn: ) failed = tool.run_all( - iter_combos, count, args.jobs, keymapdir, verbose, short, progress_bar + iter_combos, + count, + args.jobs, + keymapdir, + verbose, + short, + progress_bar, + args.chunksize, + server_socket=args.socket, + compress=args.compress, ) sys.exit(failed) diff --git a/tools/compile-keymap-server.c b/tools/compile-keymap-server.c new file mode 100644 index 00000000..e5dc8993 --- /dev/null +++ b/tools/compile-keymap-server.c @@ -0,0 +1,516 @@ +/* + * Copyright © 2024 Pierre Le Marre + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xkbcommon/xkbcommon.h" +#include "src/context.h" +#ifdef ENABLE_KEYMAP_CACHE +#include "src/xkbcomp/cache.h" +#endif +#include "tools-common.h" +#include "src/utils.h" + +#define DEFAULT_INCLUDE_PATH_PLACEHOLDER "__defaults__" + +static bool verbose = false; +static const char *includes[64]; +static size_t num_includes = 0; + +static void +usage(char **argv) +{ + printf("Usage: %s [OPTIONS]\n" + "\n" + "Start a server to compile keymaps\n" + "Options:\n" + " --help\n" + " Print this help and exit\n" + " --verbose\n" + " Enable verbose debugging output\n" + " --socket \n" + " Path of the Unix socket\n" + " --include\n" + " Add the given path to the include path list. This option is\n" + " order-dependent, include paths given first are searched first.\n" + " If an include path is given, the default include path list is\n" + " not used. Use --include-defaults to add the default include\n" + " paths\n" + " --include-defaults\n" + " Add the default set of include directories.\n" + " This option is order-dependent, include paths given first\n" + " are searched first.\n" + "\n", + argv[0]); +} + +static bool +parse_options(int argc, char **argv, struct xkb_rule_names *names, + char **socket_address) +{ + enum options { + OPT_VERBOSE, + OPT_INCLUDE, + OPT_INCLUDE_DEFAULTS, + OPT_SOCKET, + }; + static struct option opts[] = { + {"help", no_argument, 0, 'h'}, + {"verbose", no_argument, 0, OPT_VERBOSE}, + {"include", required_argument, 0, OPT_INCLUDE}, + {"include-defaults", no_argument, 0, OPT_INCLUDE_DEFAULTS}, + {"socket", required_argument, 0, OPT_SOCKET}, + {0, 0, 0, 0}, + }; + + while (1) { + int c; + int option_index = 0; + c = getopt_long(argc, argv, "h", opts, &option_index); + if (c == -1) + break; + + switch (c) { + case 'h': + usage(argv); + exit(0); + case OPT_VERBOSE: + verbose = true; + break; + case OPT_SOCKET: + *socket_address = optarg; + break; + case OPT_INCLUDE: + if (num_includes >= ARRAY_SIZE(includes)) { + fprintf(stderr, "error: too many includes\n"); + exit(EXIT_INVALID_USAGE); + } + includes[num_includes++] = optarg; + break; + case OPT_INCLUDE_DEFAULTS: + if (num_includes >= ARRAY_SIZE(includes)) { + fprintf(stderr, "error: too many includes\n"); + exit(EXIT_INVALID_USAGE); + } + includes[num_includes++] = DEFAULT_INCLUDE_PATH_PLACEHOLDER; + break; + default: + usage(argv); + exit(EXIT_INVALID_USAGE); + } + } + + return true; +} + +#define INPUT_BUFFER_SIZE 1024 + +static pthread_mutex_t server_state_mutext = PTHREAD_MUTEX_INITIALIZER; +static volatile int socket_fd; +static volatile bool serving = false; + +static void +shutdown_server(void) +{ + pthread_mutex_lock(&server_state_mutext); + fprintf(stderr, "Shuting down. Bye!\n"); + serving = false; + shutdown(socket_fd, SHUT_RD); + pthread_mutex_unlock(&server_state_mutext); +} + +static void +handle_signal(int signum) +{ + switch (signum) { + case SIGINT: + shutdown_server(); + break; + } +} + +/* Load parser for a RMLVO component */ +static bool +parse_component(char **input, size_t *max_len, const char **output) +{ + if (!(*input) || !(*max_len)) { + *input = NULL; + *max_len = 0; + *output = NULL; + return true; + } + char *start = *input; + char *next = strchr(start, '\n'); + size_t len; + if (next == NULL) { + len = *max_len; + *input = NULL; + *max_len = 0; + } else { + len = (size_t)(next - start); + *max_len -= len + 1; + *input += len + 1; + } + *output = strndup(start, len); + if (!(*output)) { + fprintf(stderr, "ERROR: Cannot allocate memory\n"); + return false; + } + return true; +} + +struct query_args { + struct xkb_context *ctx; + int accept_socket_fd; +}; + +static const char * +log_level_to_prefix(enum xkb_log_level level) +{ + switch (level) { + case XKB_LOG_LEVEL_DEBUG: + return "xkbcommon: DEBUG: "; + case XKB_LOG_LEVEL_INFO: + return "xkbcommon: INFO: "; + case XKB_LOG_LEVEL_WARNING: + return "xkbcommon: WARNING: "; + case XKB_LOG_LEVEL_ERROR: + return "xkbcommon: ERROR: "; + case XKB_LOG_LEVEL_CRITICAL: + return "xkbcommon: CRITICAL: "; + default: + return NULL; + } +} + +ATTR_PRINTF(3, 0) static void +keymap_log_fn(struct xkb_context *ctx, enum xkb_log_level level, + const char *fmt, va_list args) +{ + const char *prefix = log_level_to_prefix(level); + FILE *file = xkb_context_get_user_data(ctx); + + if (prefix) + fprintf(file, "%s", prefix); + vfprintf(file, fmt, args); +} + +/* Process client’s queries */ +static void* +process_query(void *x) +{ + struct query_args *args = x; + int rc = EXIT_FAILURE; + char input_buffer[INPUT_BUFFER_SIZE]; + + /* Loop while client send queries */ + ssize_t count; + while ((count = recv(args->accept_socket_fd, + input_buffer, INPUT_BUFFER_SIZE, 0)) > 0) { + rc = EXIT_FAILURE; + bool stop = true; + if (input_buffer[0] == '\x1b') { + /* Escape = exit */ + rc = -0x1b; + break; + } + + /* + * Expected message load: + * <1|0>\n\n\n\nvariant>\n + */ + + if (count < 3 || input_buffer[1] != '\n') { + /* Invalid length */ + break; + } + bool serialize = input_buffer[0] == '1'; + + /* We expect RMLVO to be provided with one component per line */ + char *input = input_buffer + 2; + size_t len = count - 2; + struct xkb_rule_names rmlvo = { + .rules = NULL, + .model = NULL, + .layout = NULL, + .variant = NULL, + .options = NULL, + }; + if (!parse_component(&input, &len, &rmlvo.rules) || + !parse_component(&input, &len, &rmlvo.model) || + !parse_component(&input, &len, &rmlvo.layout) || + !parse_component(&input, &len, &rmlvo.variant) || + !parse_component(&input, &len, &rmlvo.options)) { + fprintf(stderr, "ERROR: Cannor parse RMLVO: %s.\n", input_buffer); + goto error; + } + + /* Response load: */ + + /* Compile keymap */ + struct xkb_keymap *keymap; + /* Clone context because it is not thread-safe */ + struct xkb_context ctx = *args->ctx; + + /* Set our own logging function to capture default stderr */ + char *stderr_buffer = NULL; + size_t stderr_size = 0; + FILE *stderr_new = open_memstream(&stderr_buffer, &stderr_size); + if (!stderr_new) { + perror("Failed to create in-memory stderr."); + goto stderr_error; + } + xkb_context_set_user_data(&ctx, stderr_new); + xkb_context_set_log_fn(&ctx, keymap_log_fn); + + rc = EXIT_SUCCESS; + + keymap = xkb_keymap_new_from_names(&ctx, &rmlvo, XKB_KEYMAP_COMPILE_NO_FLAGS); + if (keymap == NULL) { + /* Send negative message length to convey error */ + count = -1; + send(args->accept_socket_fd, &count, sizeof(count), MSG_NOSIGNAL); + goto keymap_error; + } + + /* Send the keymap, if required */ + if (serialize) { + char *buf = xkb_keymap_get_as_string(keymap, XKB_KEYMAP_FORMAT_TEXT_V1); + if (buf) { + len = strlen(buf); + /* Send message length */ + send(args->accept_socket_fd, &len, sizeof(len), MSG_NOSIGNAL); + send(args->accept_socket_fd, buf, len, MSG_NOSIGNAL); + free(buf); + } else { + count = -1; + send(args->accept_socket_fd, &count, sizeof(count), MSG_NOSIGNAL); + } + } else { + len = 0; + send(args->accept_socket_fd, &len, sizeof(len), MSG_NOSIGNAL); + } + + xkb_keymap_unref(keymap); + +keymap_error: + /* Wait that the client confirm the reception */ + recv(args->accept_socket_fd, input_buffer, 1, 0); + + /* Restore stderr for logging */ + fflush(stderr_new); + xkb_context_set_user_data(&ctx, stderr); + fclose(stderr_new); + + /* Send captured stderr */ + send(args->accept_socket_fd, &stderr_size, sizeof(stderr_size), MSG_NOSIGNAL); + if (stderr_size && stderr_buffer) { + send(args->accept_socket_fd, stderr_buffer, stderr_size, MSG_NOSIGNAL); + } + free(stderr_buffer); + + /* Wait that the client confirm the reception */ + input_buffer[0] = '0'; + recv(args->accept_socket_fd, input_buffer, 1, 0); + stop = input_buffer[0] == '0'; + +stderr_error: +error: + free((char*)rmlvo.rules); + free((char*)rmlvo.model); + free((char*)rmlvo.layout); + free((char*)rmlvo.variant); + free((char*)rmlvo.options); + + /* Close client connection if there was an error */ + if (rc != EXIT_SUCCESS || stop) + break; + } + xkb_context_unref(args->ctx); + close(args->accept_socket_fd); + free(args); + if (rc > 0) { + fprintf(stderr, "ERROR: failed to process query. Code: %d\n", rc); + } else if (rc < 0) { + shutdown_server(); + } + return NULL; +} + +/* Create a server using Unix sockets */ +static int +serve(struct xkb_context *ctx, const char* socket_address) +{ + int rc = EXIT_FAILURE; + struct sockaddr_un sockaddr_un = { 0 }; + socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (socket_fd == -1) { + fprintf(stderr, "ERROR: Cannot create Unix socket.\n"); + return EXIT_FAILURE; + } + /* Construct the bind address structure. */ + sockaddr_un.sun_family = AF_UNIX; + strcpy(sockaddr_un.sun_path, socket_address); + + rc = bind(socket_fd, (struct sockaddr*) &sockaddr_un, + sizeof(struct sockaddr_un)); + /* If socket_address exists on the filesystem, then bind will fail. */ + if (rc == -1) { + fprintf(stderr, "ERROR: Cannot create Unix socket path.\n"); + rc = EXIT_FAILURE; + goto error_bind; + }; + if (listen(socket_fd, 4096) == -1) { + fprintf(stderr, "ERROR: Cannot listen socket.\n"); + goto error; + } + signal(SIGINT, handle_signal); + fprintf(stderr, "Serving...\n"); + serving = true; + + struct timeval timeout = { + .tv_sec = 3, + .tv_usec = 0 + }; + + while (1) { + int accept_socket_fd = accept(socket_fd, NULL, NULL); + if (accept_socket_fd == -1) { + if (serving) { + fprintf(stderr, "ERROR: fail to accept query\n"); + rc = EXIT_FAILURE; + } else { + rc = EXIT_SUCCESS; + } + goto error; + }; + + if (accept_socket_fd > 0) { + /* Client is connected */ + pthread_t thread; + + /* Prepare worker’s context */ + struct query_args *args = calloc(1, sizeof(struct query_args)); + if (!args) { + close(accept_socket_fd); + continue; + } + /* Context will be cloned in worker */ + args->ctx = xkb_context_ref(ctx); + args->accept_socket_fd = accept_socket_fd; + + if (setsockopt(accept_socket_fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof timeout) < 0) + perror("setsockopt failed\n"); + + /* Launch worker */ + rc = pthread_create(&thread, NULL, process_query, args); + if (rc) { + perror("Error creating thread: "); + close(accept_socket_fd); + free(args); + continue; + } + pthread_detach(thread); + } + } +error: + close(socket_fd); + unlink(socket_address); +error_bind: + fprintf(stderr, "Exiting...\n"); + return rc; +} + +int +main(int argc, char **argv) +{ + struct xkb_context *ctx; + struct xkb_rule_names names = { + .rules = DEFAULT_XKB_RULES, + .model = DEFAULT_XKB_MODEL, + /* layout and variant are tied together, so we either get user-supplied for + * both or default for both, see below */ + .layout = NULL, + .variant = NULL, + .options = DEFAULT_XKB_OPTIONS, + }; + char *socket_address = NULL; + int rc = EXIT_FAILURE; + + if (argc < 1) { + usage(argv); + return EXIT_INVALID_USAGE; + } + + if (!parse_options(argc, argv, &names, &socket_address)) + return EXIT_INVALID_USAGE; + + ctx = xkb_context_new(XKB_CONTEXT_NO_DEFAULT_INCLUDES); + assert(ctx); + + if (verbose) { + xkb_context_set_log_level(ctx, XKB_LOG_LEVEL_DEBUG); + xkb_context_set_log_verbosity(ctx, 10); + } + + if (num_includes == 0) + includes[num_includes++] = DEFAULT_INCLUDE_PATH_PLACEHOLDER; + + for (size_t i = 0; i < num_includes; i++) { + const char *include = includes[i]; + if (strcmp(include, DEFAULT_INCLUDE_PATH_PLACEHOLDER) == 0) + xkb_context_include_path_append_default(ctx); + else + xkb_context_include_path_append(ctx, include); + } + +#ifdef ENABLE_KEYMAP_CACHE + struct xkb_keymap_cache *keymap_cache = xkb_keymap_cache_new(); + if (keymap_cache) + ctx->keymap_cache = keymap_cache; + else + fprintf(stderr, "ERROR: Cannot create keymap cache.\n"); +#endif + serve(ctx, socket_address); +#ifdef ENABLE_KEYMAP_CACHE + xkb_keymap_cache_free(keymap_cache); +#endif + + xkb_context_unref(ctx); + + return rc; +} From 22dd336a9bcc948986b683bc50e970b7e154bf8f Mon Sep 17 00:00:00 2001 From: Pierre Le Marre Date: Thu, 14 Nov 2024 21:30:55 +0100 Subject: [PATCH 7/7] fix atomic types --- src/atom.c | 2 +- src/context.h | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/atom.c b/src/atom.c index e8632a30..7635652f 100644 --- a/src/atom.c +++ b/src/atom.c @@ -200,7 +200,7 @@ atom_intern(struct atom_table *table, const char *string, size_t len, bool add) struct atom_table { size_t size; - char **strings; + _Atomic(char *) *strings; }; struct atom_table * diff --git a/src/context.h b/src/context.h index 51c9db32..6379d18f 100644 --- a/src/context.h +++ b/src/context.h @@ -26,6 +26,8 @@ #ifndef CONTEXT_H #define CONTEXT_H +#include + #include "xkbcommon/xkbcommon.h" #include "atom.h" #include "messages-codes.h" @@ -35,7 +37,7 @@ #endif struct xkb_context { - int refcnt; + atomic_int refcnt; ATTR_PRINTF(3, 0) void (*log_fn)(struct xkb_context *ctx, enum xkb_log_level level,