Skip to content

Commit

Permalink
support for files without external meta data (#12)
Browse files Browse the repository at this point in the history 
Add support for reading the meta data inside a squashfs image.

Calls the external `unsquashfs` application to unpack the `meta` path inside a squashfs image to a temporary working directory.
  • Loading branch information
@bcumming
bcumming authored Oct 16, 2024
1 parent 175ed37 commit cb0d9f5
Show file tree
Hide file tree
Showing 10 changed files with 377 additions and 56 deletions.
4 changes: 3 additions & 1 deletion meson.build
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,11 @@ lib_src = [
'src/uenv/meta.cpp',
'src/uenv/parse.cpp',
'src/uenv/repository.cpp',
'src/uenv/uenv.cpp',
'src/util/fs.cpp',
'src/util/shell.cpp',
'src/util/strings.cpp',
'src/util/subprocess.cpp',
'src/uenv/uenv.cpp',
]
lib_inc = include_directories('src')

Expand Down Expand Up @@ -76,6 +77,7 @@ endif
unit_src = [
'test/unit/env.cpp',
'test/unit/envvars.cpp',
'test/unit/fs.cpp',
'test/unit/lex.cpp',
'test/unit/main.cpp',
'test/unit/parse.cpp',
Expand Down
171 changes: 123 additions & 48 deletions src/uenv/env.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,62 @@
#include <uenv/meta.h>
#include <uenv/parse.h>
#include <uenv/repository.h>
#include <util/fs.h>
#include <util/subprocess.h>

namespace uenv {

using util::unexpected;

struct meta_info {
std::optional<std::filesystem::path> path;
std::optional<std::filesystem::path> env;
};

meta_info find_meta_path(const std::filesystem::path& sqfs_path) {
namespace fs = std::filesystem;

meta_info meta;
if (fs::is_directory(sqfs_path.parent_path() / "meta")) {
meta.path = sqfs_path.parent_path() / "meta";
spdlog::debug("find_meta_path: {} found external meta path {}",
sqfs_path.string(), meta.path->string());
} else if (auto p = util::unsquashfs_tmp(sqfs_path, "meta")) {
meta.path = *p / "meta";
spdlog::debug("find_meta_path: {} found internal meta path {}",
sqfs_path.string(), meta.path->string());
} else {
spdlog::debug("find_meta_path: {} no meta path found",
sqfs_path.string());
}

if (meta.path) {
auto env_meta = meta.path.value() / "env.json";

if (fs::is_regular_file(env_meta)) {
meta.env = env_meta;
spdlog::debug("find_meta_path: {} found env meta {}",
sqfs_path.string(), meta.env->string());
} else {
spdlog::debug("find_meta_path: {} no env meta", sqfs_path.string());
}
}

return meta;
}

util::expected<env, std::string>
concretise_env(const std::string& uenv_args,
std::optional<std::string> view_args,
std::optional<std::filesystem::path> repo_arg) {
namespace fs = std::filesystem;

// parse the uenv description that was provided as a command line argument.
// the command line argument is a comma-separated list of uenvs, where each
// uenv is either
// parse the uenv description that was provided as a command line
// argument. the command line argument is a comma-separated list of
// uenvs, where each uenv is either
// - the path of a squashfs image; or
// - a uenv description of the form name[/version][:tag][@system][%uarch]
// with an optional mount point.
// - a uenv description of the form
// name[/version][:tag][@system][%uarch] with an optional mount point.
const auto uenv_descriptions = uenv::parse_uenv_args(uenv_args);
if (!uenv_descriptions) {
return unexpected(fmt::format("invalid uenv description: {}",
Expand All @@ -38,8 +77,8 @@ concretise_env(const std::string& uenv_args,

// concretise the uenv descriptions by looking for the squashfs file, or
// looking up the uenv descrition in a registry.
// after this loop, we have fully validated list of uenvs, mount points and
// meta data (if they have meta data).
// after this loop, we have fully validated list of uenvs, mount points
// and meta data (if they have meta data).

std::unordered_map<std::string, uenv::concrete_uenv> uenvs;
std::set<fs::path> used_mounts;
Expand All @@ -52,9 +91,10 @@ concretise_env(const std::string& uenv_args,
// it has to be looked up in a repo.
if (auto label = desc.label()) {
if (!repo_arg) {
return unexpected(
"a repo needs to be provided either using the --repo flag "
"or by setting the UENV_REPO_PATH environment variable");
return unexpected("a repo needs to be provided either "
"using the --repo flag "
"or by setting the UENV_REPO_PATH "
"environment variable");
}
auto store = uenv::open_repository(*repo_arg);
if (!store) {
Expand Down Expand Up @@ -119,22 +159,22 @@ concretise_env(const std::string& uenv_args,
}
spdlog::info("{} squashfs image {}", desc, sqfs_path.string());

// set the meta data path and env.json path if they exist
const auto meta_p = sqfs_path.parent_path() / "meta";
const auto env_meta_p = meta_p / "env.json";
const std::optional<fs::path> meta_path =
fs::is_directory(meta_p) ? meta_p : std::optional<fs::path>{};
const std::optional<fs::path> env_meta_path =
fs::is_regular_file(env_meta_p) ? env_meta_p
: std::optional<fs::path>{};

// if meta/env.json exists, parse the json therein
std::string name;
std::string description;
// create a default "anonymous" name for the uenv that will be
// overwritten if meta data is provided.
std::string name = "anonymous";
unsigned name_idx = 0;
while (uenvs.count(name)) {
name = fmt::format("anonymous{}", name_idx);
++name_idx;
}
std::string description = "";
std::optional<std::string> mount_meta;
std::unordered_map<std::string, concrete_view> views;
if (env_meta_path) {
if (const auto result = uenv::load_meta(*env_meta_path)) {

// if meta/env.json exists, parse the json therein
auto meta = find_meta_path(sqfs_path);
if (meta.env) {
if (const auto result = uenv::load_meta(*(meta.env))) {
name = std::move(result->name);
description = result->description.value_or("");
mount_meta = result->mount;
Expand All @@ -143,27 +183,19 @@ concretise_env(const std::string& uenv_args,
mount_meta);
} else {
spdlog::warn("{} opening the uenv meta data {}: {}", desc,
*env_meta_path, result.error());
meta.env->string(), result.error());
}
} else {
spdlog::debug("{} no meta file found at expected location {}", desc,
meta_path);
description = "";
// generate a unique name for the uenv
name = "anonymous";
unsigned i = 1;
while (uenvs.count(name)) {
name = fmt::format("anonymous{}", i);
++i;
}
spdlog::warn("{} no meta file available for {}", desc,
sqfs_path.string());
}

// if an explicit mount point was provided, use that
// otherwise use the mount point provided in the meta data
auto mount_string = desc.mount() ? desc.mount() : mount_meta;

// handle the case where no mount point was provided by the CLI or meta
// data
// handle the case where no mount point was provided by the CLI or
// meta data
if (!mount_string) {
return unexpected(
fmt::format("no mount point provided for {}", desc));
Expand Down Expand Up @@ -212,7 +244,7 @@ concretise_env(const std::string& uenv_args,
}

uenvs[name] = concrete_uenv{name, mount, sqfs_path,
meta_path, description, std::move(views)};
meta.path, description, std::move(views)};
}

// A dictionary with view name as a key, and a list of uenv that provide
Expand Down Expand Up @@ -249,7 +281,8 @@ concretise_env(const std::string& uenv_args,
// a list of uenv that have a view with name v.name
const auto& matching_uenvs = view2uenv[view.name];

// handle the case where no uenv name was provided, e.g. develop
// handle the case where no uenv name was provided, e.g.
// develop
if (!view.uenv) {
// it is ambiguous if more than one option is available
if (matching_uenvs.size() > 1) {
Expand All @@ -263,8 +296,8 @@ concretise_env(const std::string& uenv_args,
}
views.push_back({matching_uenvs[0], view.name});
}
// handle the case where both uenv and view name are provided,
// e.g. prgenv-gnu:develop
// handle the case where both uenv and view name are
// provided, e.g. prgenv-gnu:develop
else {
auto it = std::find_if(
matching_uenvs.begin(), matching_uenvs.end(),
Expand Down Expand Up @@ -298,19 +331,18 @@ std::unordered_map<std::string, std::string> getenv(const env& environment) {

// returns the value of an environment variable.
// if the variable has been recorded in env_vars, that value is returned
// else the cstdlib getenv function is called to get the currently set value
// returns nullptr if the variable is not set anywhere
// else the cstdlib getenv function is called to get the currently set
// value returns nullptr if the variable is not set anywhere
auto ge = [&env_vars](const std::string& name) -> const char* {
if (env_vars.count(name)) {
return env_vars[name].c_str();
}
return ::getenv(name.c_str());
};

// iterate over each view in order, and set the environment variables that
// each view configures.
// the variables are not set directly, instead they are accumulated in
// env_vars.
// iterate over each view in order, and set the environment variables
// that each view configures. the variables are not set directly,
// instead they are accumulated in env_vars.
for (auto& view : environment.views) {
auto result = environment.uenvs.at(view.uenv)
.views.at(view.name)
Expand All @@ -323,11 +355,53 @@ std::unordered_map<std::string, std::string> getenv(const env& environment) {
return env_vars;
}

// list of environment variables that are ignored in setuid applications
// the full list is defined here:
// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/generic/unsecvars.h
std::set<std::string_view> unsecure_envvars__{
"GCONV_PATH",
"GETCONF_DIR",
"GLIBC_TUNABLES",
"HOSTALIASES",
"LD_AUDIT",
"LD_BIND_NOT",
"LD_BIND_NOW",
"LD_DEBUG",
"LD_DEBUG_OUTPUT",
"LD_DYNAMIC_WEAK",
"LD_LIBRARY_PATH",
"LD_ORIGIN_PATH",
"LD_PRELOAD",
"LD_PROFILE",
"LD_SHOW_AUXV",
"LD_VERBOSE",
"LD_WARN",
"LOCALDOMAIN",
"LOCPATH",
"MALLOC_ARENA_MAX",
"MALLOC_ARENA_TEST",
"MALLOC_MMAP_MAX_",
"MALLOC_MMAP_THRESHOLD_",
"MALLOC_PERTURB_",
"MALLOC_TOP_PAD_",
"MALLOC_TRACE",
"MALLOC_TRIM_THRESHOLD_",
"NIS_PATH",
"NLSPATH",
"RESOLV_HOST_CONF",
"RES_OPTIONS",
"TMPDIR",
};

util::expected<int, std::string>
setenv(const std::unordered_map<std::string, std::string>& variables,
const std::string& prefix) {
for (auto var : variables) {
std::string fwd_name = prefix + var.first;
// prepend prefix to unsecure environment variables
std::string fwd_name = unsecure_envvars__.contains(var.first)
? prefix + var.first
: var.first;
fmt::println("setting {} to {}", fwd_name, var.second);
if (auto rcode = ::setenv(fwd_name.c_str(), var.second.c_str(), true)) {
switch (rcode) {
case EINVAL:
Expand All @@ -340,6 +414,7 @@ setenv(const std::unordered_map<std::string, std::string>& variables,
fmt::format("unknown error setting {}", fwd_name));
}
}
fmt::println("set!");
}
return 0;
}
Expand Down
Loading

0 comments on commit cb0d9f5

Please sign in to comment.