Skip to content

Commit

Permalink
a quick(maybe and dirty) impletion of RFE: issue 228 and more
Browse files Browse the repository at this point in the history 
see: restraint-harness#288
and support customize the task fetch options in param: _fetch_opts:
retry=N
timeo=N
keepchanges
abort_recipe|abort_recipe_when_fetch_fail

e.g: If the key task /kernel/install fetch fails, subsequent tasks
will not run:
<task name="/kernel/install">
  <fetch url="http://a.b.c/path/archive.tgz#rpath"/>
  <params>
    <param name="_fetch_opts" value="retry=16,timeo=16,abort_recipe"/>
    <param name="KERNEL_VER" value="kernel-6.7.0-119.el10"/>
  </params>
</task>
<task name="/kernel/fs/nfs/test1">
  <fetch url="http://a.b.c/path/archive.tgz#rpath1"/>
  <params>
    <param name="_fetch_opts" value="retry=8,keepchanges"/>
    <param name="KERNEL_VER" value="kernel-6.7.0-119.el10"/>
  </params>
</task>

Q: why add these options in task params?
A: because beaker job xml schema has not accept the new xml tag,
so as a workaround I put these fetch options in task->params

Signed-off-by: Jianhong Yin <[email protected]>
  • Loading branch information
@tcler
tcler committed May 7, 2024
1 parent 8b2b213 commit 4a95e7a
Show file tree
Hide file tree
Showing 6 changed files with 131 additions and 4 deletions.
1 change: 1 addition & 0 deletions restraint.spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,7 @@ fi
%attr(0755, root, root)%{_bindir}/rstrnt-abort
%attr(0755, root, root)%{_bindir}/rstrnt-sync
%attr(0755, root, root)%{_bindir}/rstrnt-package
%attr(0755, root, root)%{_bindir}/rstrnt-set-abort-recipe
/usr/share/%{name}/plugins/run_plugins
/usr/share/%{name}/plugins/run_task_plugins
/usr/share/%{name}/plugins/helpers
Expand Down
4 changes: 3 additions & 1 deletion scripts/Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
FILES := rstrnt-backup rstrnt-restore rstrnt-prepare-reboot rstrnt-reboot check_beaker rstrnt-package rstrnt-sync-set rstrnt-sync-block
FILES := rstrnt-backup rstrnt-restore rstrnt-prepare-reboot rstrnt-reboot check_beaker rstrnt-package rstrnt-sync-set \
rstrnt-sync-block rstrnt-set-abort-recipe


.PHONY: all
all:
Expand Down
7 changes: 7 additions & 0 deletions scripts/rstrnt-set-abort-recipe
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/bash

#create file "/tmp/abort_recipe_$RECIPEID" to notify subsequent tasks in same
#recipe to abort without do anything.
#restraintd program will check whether this file exists before prepare each task
#if exists then quit immediately.
touch /tmp/abort_recipe_${RECIPEID:-$RSTRNT_RECIPEID}
63 changes: 62 additions & 1 deletion src/recipe.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,47 @@ static GList *parse_roles(xmlNode *roles_node, GError **error) {
return NULL;
}

static void parse_fetch_options(gchar *fetch_opts, Task *task)
{
gint i, j;
gchar **opts, **optsj, *token, *tokenj;

if (fetch_opts == NULL)
return;

opts = g_strsplit(fetch_opts, " ", 0);
for (i = 0; token = opts[i], token != NULL; i++) {
optsj = g_strsplit(token, ",", 0);
for (j = 0; tokenj = optsj[j], tokenj != NULL; j++) {
if (0 == strncmp("retry=", tokenj, 6)) {
task->fetch_retries = atoi(&tokenj[6]);
if (task->fetch_retries <= 0) {
task->fetch_retries = TASK_FETCH_RETRIES;
}
} else if (0 == strncmp("timeo=", tokenj, 6)) {
task->fetch_interval = atoi(&tokenj[6]);
if (task->fetch_interval <= 0) {
task->fetch_interval = TASK_FETCH_INTERVAL;
}
} else if (0 == strncmp("abort_recipe_when_fetch_fail", tokenj, 5)) {
task->abort_recipe_when_fetch_fail = TRUE;
} else if (0 == strncmp("keepchanges", tokenj, 11)) {
task->keepchanges = TRUE;
}
}
g_strfreev(optsj);
}
g_strfreev(opts);
}

static void check_param_fetch_opts(Param *param, Task *task)
{
g_return_if_fail (param != NULL && task != NULL);
if (STREQ(param->name, "_fetch_opts")) {
parse_fetch_options(param->value, task);
}
}

static Task *parse_task(xmlNode *task_node, Recipe *recipe, GError **error) {
g_return_val_if_fail(task_node != NULL, NULL);
g_return_val_if_fail(error == NULL || *error == NULL, NULL);
Expand All @@ -208,6 +249,8 @@ static Task *parse_task(xmlNode *task_node, Recipe *recipe, GError **error) {
g_return_val_if_fail(task != NULL, NULL);

task->recipe = recipe;
task->fetch_retries = TASK_FETCH_RETRIES;
task->fetch_interval = TASK_FETCH_INTERVAL;
xmlChar *task_name = xmlGetNoNsProp(task_node, (xmlChar *)"name");
task->name = g_strdup((gchar *)task_name);
xmlFree (task_name);
Expand All @@ -218,6 +261,12 @@ static Task *parse_task(xmlNode *task_node, Recipe *recipe, GError **error) {
}
xmlFree(keepchanges);

xmlChar *fetch_opts = xmlGetNoNsProp(task_node, (xmlChar *)"fetch_opts");
if (fetch_opts != NULL) {
parse_fetch_options((gchar *)fetch_opts, task);
xmlFree(fetch_opts);
}

xmlChar *task_id = xmlGetNoNsProp(task_node, (xmlChar *)"id");
if (task_id == NULL) {
unrecognised("<task/> without id");
Expand All @@ -239,8 +288,8 @@ static Task *parse_task(xmlNode *task_node, Recipe *recipe, GError **error) {
task->task_id);
goto error;
}
task->fetch.url = soup_uri_new((char *)url);

task->fetch.url = soup_uri_new((char *)url);
if (task->fetch.url == NULL) {
unrecognised("'%s' from task %s is not a valid url", url,
task->task_id);
Expand Down Expand Up @@ -291,6 +340,7 @@ static Task *parse_task(xmlNode *task_node, Recipe *recipe, GError **error) {
xmlNode *params_node = first_child_with_name(task_node, "params", FALSE);
if (params_node != NULL) {
task->params = parse_params(params_node, &tmp_error);
g_list_foreach (task->params, (GFunc)check_param_fetch_opts, task);
/* params could be empty, but if parsing causes an error then fail */
if (tmp_error != NULL) {
g_propagate_prefixed_error(error, tmp_error,
Expand Down Expand Up @@ -402,6 +452,14 @@ void restraint_recipe_free(Recipe *recipe) {
g_slice_free(Recipe, recipe);
}

static void
clear_abort_recipe(Recipe *recipe)
{
char cmd[PATH_MAX] = { 0 };
snprintf(cmd, PATH_MAX-1, "rm -f /tmp/abort_recipe_%s", recipe->recipe_id);
(void)!system(cmd);
}

static Recipe *
recipe_parse (xmlDoc *doc, SoupURI *recipe_uri, GError **error, gchar **cfg_file)
{
Expand Down Expand Up @@ -433,6 +491,9 @@ recipe_parse (xmlDoc *doc, SoupURI *recipe_uri, GError **error, gchar **cfg_file
result->osvariant = get_attribute(recipe, "variant");
result->owner = get_attribute(job, "owner");

/* remove possible abort_recipe_<recipe-id> file at recipe beginning */
clear_abort_recipe(result);

if (recipe_uri == NULL) {
gchar *tmp_str;

Expand Down
51 changes: 49 additions & 2 deletions src/task.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,24 @@ refresh_role_retry (gpointer user_data)
return FALSE;
}

static void
set_abort_recipe(Task *task)
{
char cmd[PATH_MAX] = { 0 };
snprintf(cmd, PATH_MAX-1, "echo %s_fetch-fail >/tmp/abort_recipe_%s",
task->name, task->recipe->recipe_id);
(void)!system(cmd);
}

static gboolean
get_abort_recipe(Task *task)
{
gchar abort_recipe_file[PATH_MAX] = { 0 };
snprintf(abort_recipe_file, PATH_MAX-1, "/tmp/abort_recipe_%s",
task->recipe->recipe_id);
return g_file_test(abort_recipe_file, G_FILE_TEST_EXISTS);
}

void
fetch_finish_callback (GError *error, guint32 match_cnt,
guint32 nonmatch_cnt, gpointer user_data)
Expand All @@ -107,15 +125,24 @@ fetch_finish_callback (GError *error, guint32 match_cnt,
GString *message = g_string_new (NULL);

if (error) {
if (app_data->fetch_retries < TASK_FETCH_RETRIES) {
if (app_data->fetch_retries < task->fetch_retries) {
g_warning("* RETRY fetch [%d]**:%s\n", ++app_data->fetch_retries,
error->message);
g_clear_error(&error);
g_timeout_add_seconds (TASK_FETCH_INTERVAL, fetch_retry, app_data);
g_timeout_add_seconds (task->fetch_interval, fetch_retry, app_data);
return;
} else {
g_propagate_error (&task->error, error);
task->state = TASK_COMPLETE;

/*
* for issue github.com/restraint-harness/restraint/issues/288
* if user declared in task <fetch url="$url abort[_recipe_when_fetch_fail]">
* create file: /tmp/abort_recipe_$recipe_id to notify subsequent tasks
*/
if (task->abort_recipe_when_fetch_fail) {
set_abort_recipe(task);
}
}
} else {
task->state = TASK_METADATA_PARSE;
Expand All @@ -141,6 +168,18 @@ restraint_task_fetch(AppData *app_data) {
GError *error = NULL;
Task *task = (Task *) app_data->tasks->data;

/* Abandon subsequent tasks, if critical task fetching or execution fails */
if (get_abort_recipe(task)) {
g_set_error (&error, RESTRAINT_ERROR,
RESTRAINT_TASK_RUNNER_ABORTED,
"Task for test-prepare fetch fail");
task->fetch_retries = 0;
task->fetch_interval = 0.1;
fetch_finish_callback (error, 0, 0, app_data);
g_return_if_reached();
return;
}

switch (task->fetch_method) {
case TASK_FETCH_UNPACK:
{
Expand Down Expand Up @@ -438,6 +477,14 @@ void dependency_finish_cb (gpointer user_data, GError *error)
if (error) {
g_propagate_error(&task->error, error);
task->state = TASK_COMPLETE;
/*
* for issue github.com/restraint-harness/restraint/issues/288
* if user declared in task <fetch url="$url abort[_recipe_when_fetch_fail]">
* create file: /tmp/abort_recipe_$recipe_id to notify subsequent tasks
*/
if (task->abort_recipe_when_fetch_fail) {
set_abort_recipe(task);
}
} else {
task->state = TASK_RUN;
}
Expand Down
9 changes: 9 additions & 0 deletions src/task.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@
#define TASK_FETCH_INTERVAL 20
#define TASK_FETCH_RETRIES 30

#ifndef PATH_MAX
#define PATH_MAX 512
#endif

typedef enum {
TASK_IDLE,
TASK_FETCH,
Expand Down Expand Up @@ -84,6 +88,11 @@ typedef struct RstrntTask {
gchar *package_name; // when TASK_FETCH_INSTALL_PACKAGE
SoupURI *url; // when TASK_FETCH_UNPACK
} fetch;
/* Max retry number and timeout before give up fetching */
int fetch_retries;
int fetch_interval;
/* Flag indicating that abort the entire recipe if fetch fail */
gboolean abort_recipe_when_fetch_fail;
/* Whether to keep task changes */
gboolean keepchanges;
gboolean ssl_verify;
Expand Down

0 comments on commit 4a95e7a

Please sign in to comment.