apt.cf

#
## apt.cf @SURF
#
bundle common apt
{
    vars:
        any::
            "check_command" string => "/usr/bin/apt-get check";
            "config_dir" string => "/etc/apt";
            "dpkg_command" string => "/usr/bin/dpkg";
            "dpkg_reconfigure"string => "/usr/sbin/dpkg-reconfigure";
            "non_free_component" string => "non-free-firmware";
            "repos_dir" string => "$(config_dir)/sources.list.d";
            "trusted_dir" string => "$(config_dir)/trusted.gpg.d";

            "packages"  data => parsejson('{
                "install": {
                    "debconf-utils": "",
                    "apt-transport-https": "",
                    "dirmngr": ""
                }
            }');

        debian_6|debian_7::
            "pkg_command" string => "/usr/bin/apt-get";

        !(debian_6|debian_7)::
            # Not stable yet 5 Apr 2018
            #"pkg_command" string => "/usr/bin/apt";
            "pkg_command" string => "/usr/bin/apt-get";

        debian_6::
            "os_name" string => "squeeze";
        debian_7::
            "os_name" string => "wheezy";
        debian_8::
            "os_name" string => "jessie";
        debian_9::
            "os_name" string => "stretch";
            "non_free_component" string => "non-free";
        debian_10::
            "os_name" string => "buster";
            "non_free_component" string => "non-free";
        debian_11::
            "os_name" string => "bullseye";
            "non_free_component" string => "non-free";
        debian_12::
            "os_name" string => "bookworm";
}

bundle agent apt_autorun()
{
    meta:
        "tags"  slist => { "service_apt", "template_apt" };

    vars:
        APT_OS_VERSION_CHECK::
            "os_updates" string => "on";
        !APT_OS_VERSION_CHECK::
            "os_updates" string => "off";
        APT_AUTOMATIC_SECURITY_UPDATE::
            "security_updates" string => "on";
        !APT_AUTOMATIC_SECURITY_UPDATE::
            "security_updates" string => "off";

    classes:
        "apt_debconf_set" expression => isvariable("scl.apt[debconf]"),
            comment => "Must we modify debconf settings";

    methods:
        any::
            "" usebundle => apt_install();
            "" usebundle => apt_repository();
            "" usebundle => apt_auth_conf();

        APT_AUTOREMOVE::
            "" usebundle => apt_autoremove();

        APT_AUTOMATIC_SECURITY_UPDATE::
            "" usebundle => apt_security_update_silent(),
                    comment => "Only run once a hour due to if_elapsed",
                    if => "$(scl.apt[security_schedule])",
                    action => if_elapsed(50);

        APT_CHECK_STATUS::
            "" usebundle => apt_check_status();

        APT_DIST_UPGRADE::
            "" usebundle => apt_package_update("-y dist-upgrade");

        APT_KILL_PKG_MANAGER::
            "" usebundle => apt_kill_pkg_manager();

        APT_OS_VERSION_CHECK::
            "" usebundle => apt_os_version();

        APT_SYSTEMD_DISABLE::
            "" usebundle => apt_systemd_disable(),
                    comment => "Only run once a day",
                    action => if_elapsed_day;

        Security::
            "" usebundle => apt_security_update(""),
                comment => "set via de command line,i -DSecurity";

        Upgrade::
            "" usebundle => apt_package_update("dist-upgrade"),
                comment => "set via de command line, -DUpgrade";

        # Must be ported to def.json
        apt_debconf_set::
            "" usebundle => apt_debconf(),
                if => "$(scl.apt[debconf_schedule])|APT_DEBCONF";

    reports:
        DEBUG|DEBUG_apt::
            "$(this.bundle): has been called and automatic security update is: $(security_updates)";
            "$(this.bundle): has been called and automatic os update is: $(os_updates)";
}

bundle agent apt_autoremove()
{
    commands:
        any::
            "$(apt.pkg_command)"
                args => "-y autoremove",
                if => "$(scl.apt[autoremove_schedule])";
}

bundle agent apt_kill_pkg_manager()
{
    vars:
        "apt_cmds" slist => { "apt-get", "aptitude" };

    methods:
        debian::
            "" usebundle => scl_kill_process("$(apt_cmds)",1),
                classes =>  results("bundle", "$(this.bundle)_$(apt_cmds)" );

    reports:
        "$(this.bundle): $(apt_cmds) was killed on host $(sys.host) running longer then a hour"
            if => or( canonify("$(this.bundle)_$(apt_cmds)_repaired") );
}

bundle agent apt_os_version()
{
    vars:
        debian::
            "os_class_list" slist => classesmatching("debian.*");
            "os_class_string" string => format("%S", os_class_list);

    classes:
        any::
            "flavor_set"  expression => isvariable("def.apt[$(sys.flavor)]");

        flavor_set::
            "version_ok" expression => reglist(@(os_class_list), "$(def.apt[$(sys.flavor)])");

    methods:
        !version_ok.flavor_set::
            "" usebundle => apt_package_update("-y dist-upgrade");


    reports:
        DEBUG_apt.flavor_set::
            "$(this.bundle): $(sys.flavor) version is set in def.cf to: $(def.apt[$(sys.flavor)])";

        !flavor_set::
            "$(this.bundle): AUTOMATIC OS update failed due to missing setting in def.cf/json file, eg:";
            "   apt: { $(sys.flavor): $(sys.flavor)_4 }";

        flavor_set.!version_ok::
            "$(this.bundle): Debian version MISMATCH: $(def.apt[$(sys.flavor)]) not in $(os_class_string)";
}


bundle agent apt_systemd_disable()
{
    vars:
        any::
            "systemd_services_timers" slist => { "apt-daily.timer", "apt-daily-upgrade.timer" };

    methods:
        debian_9::
            "" usebundle => systemd_stop_service("$(systemd_services_timers)");

    commands:
        "$(paths.path[systemctl]) stop $(systemd_services_timers)"
            if => canonify("systemd_service_$(systemd_services_timers)_repaired");
}


bundle agent apt_package_update(update_option)
{
    commands:
        debian::
            "$(apt.dpkg_command)"   args => "--configure -a";
            "$(apt.pkg_command)"    args => "--quiet update";
            "$(apt.pkg_command)"    args => "$(update_option)";
            "$(apt.pkg_command)"    args => "clean";

    reports:
        DEBUG_apt.debian::
            "$(this.bundle): '$(sys.host)':'$(options)'";

}

bundle agent apt_security_update_silent()
{
    methods:
        any::
            "" usebundle => apt_security_update("-qq"),
                comment => "Show only the packages that are updated",
                action => immediate;
}

bundle agent apt_security_update(options)
{

    vars:
        debian::
            "apt_return_codes" slist => { "1" , "2" , "100" };
            "dpkg_options" slist => maplist("-o $(this)", getvalues("scl.apt[security_options]"));
            "dpkg_options_str" string => join(" ", dpkg_options);

    commands:
        debian::
            "$(apt.pkg_command) --quiet update"
                comment => "Update the repository indexes else we get out of sync",
                contain => in_shell_and_silent,
                classes => scl_cmd_repair( "@(apt_return_codes)",  "apt_update");

            "$(apt.pkg_command) -y upgrade"
                comment => "New security update command",
                contain => in_shell,
                args => "$(dpkg_options_str) $(options)",
                classes => scl_cmd_repair( "@(apt_return_codes)",  "apt_upgrade");

        apt_upgrade_failed|apt_update_failed::
            "$(apt.pkg_command) --fix-broken install"
                comment => "Fix the apt",
                contain => in_shell;
            "$(apt.dpkg_command) --configure -a"
                comment => "Fix dpkg interruption",
                contain => in_shell;

    reports:
        apt_update_failed::
            "$(this.bundle): apt update FAILED on host  $(sys.host), running commands to fix it ";

        apt_upgrade_failed::
            "$(this,bundle): apt -y upgrade FAILED on host  $(sys.host), running commands to fix it";

        DEBUG_apt|Security::
            "$(this.bundle): has run with the dpkg options: '$(dpkg_options_str)'";
}

bundle agent apt_check_status()
{
    vars:
        "bad_state" slist => { "100" };

    commands:
        debian::
            "$(apt.check_command)"
                comment => "Check status of the APT package manager",
                contain => in_shell_and_silent,
                classes => scl_cmd_repair("@(bad_state)", "apt_check_status");

        apt_check_status_failed::
            "$(apt.dpkg_command)"
                args => "--configure -a",
                comment => "Try to fix the bad apt status";

    reports:
        any::
            "$(this.bundle): Checking package manager status (apt)"
                if => "DEBUG|DEBUG_apt|DEBUG_$(this.bundle)";

        apt_check_status_failed::
            "$(this.bundle): ERROR! Package manager returned an error!";
}

bundle agent apt_repository()
{
    vars:
        any::
            "template_file" string => "repository.mustache";
            "template_list" slist => { "$(template_file)" };

        !apt_repository_json_files_set::
            "repo_files" slist => { getvalues("scl.apt[repo_files]") };
        apt_repository_json_files_set::
            "repo_files" slist => { @(def.apt_repo_files), getvalues("scl.apt[repo_files]") },
                comment => "Support old behaviour till we converted everything to new apt setup";

    classes:
        "apt_repository_json_files_set" expression => isvariable("def.apt_repo_files"),
                comment => "Support old behaviour till we converted everything to new apt setup";

    methods:
        any::
            "" usebundle => scl_json_copy("apt", "@(repo_files)");
            "" usebundle => scl_mustache_copy("apt", "@(template_list)");
            "" usebundle => apt_repository_generate("$(template_file)", "@(repo_files)");

    reports:
        (DEBUG|DEBUG_apt|DEBUG_apt_repository)::
            "$(this.bundle): Specified repository files: $(repo_files)";

}

bundle agent apt_repository_generate(template_file, repo_files)
{
    vars:
        any::
            "json_dir" string => "$(def.node_template_dir)/apt";

            "repo_data" data => parsejson("{}");
            "repo_data" data => mergedata("repo_data", readjson("$(json_dir)/$(repo_files)"));
            "repo_data" data => data_expand("repo_data");

            "repo_data_str" string => format("%S", "repo_data"),
                if => "DEBUG|DEBUG_apt|DEBUG_apt_repository";

            "repo_names" slist => getindices("repo_data");
            "repo_exclude_files" slist => maplist("$(this).list", @(repo_names));

        DEBUG|DEBUG_apt|DEBUG_apt_repository::
            "repo_exclude_files_str" string => format("%S", repo_exclude_files);

    classes:
        any::
            "apt_import_key_cmd_exists" expression => fileexists("$(scl.apt[import_key])");

    files:
        any::
            "$(apt.repos_dir)/$(repo_names).list"
                create          => "true",
                edit_template   => "/etc/node_status/templates/apt/$(template_file)",
                template_method => "mustache",
                template_data   => mergedata("repo_data[$(repo_names)]");

            "$(apt.repos_dir)"
                comment         => "Remove apt repository files that are not maintained by CFEngine",
                delete          => scl_tidyfiles,
                file_select     => ex_list("@(repo_exclude_files)"),
                depth_search    => recurse("inf");

            "$(apt.trusted_dir)/$(repo_data[$(repo_names)][key_file])"
                comment => "Must we copy repo key file",
                move_obstructions => "true",
                perms => mog("0644", "root", "root"),
                copy_from   => remote_dcp("$(scl.apt[key_dir])/$(repo_data[$(repo_names)][key_file])", "$(sys.policy_hub)");

        APT_SOURCES_FILE_REMOVE::
            "/etc/apt/sources.list"
                comment         => "Remove previously used (link) sources.list file",
                delete          => tidy;

    methods:
        apt_import_key_cmd_exists::
            "" usebundle => apt_repository_key("$(repo_names)", "$(repo_data[$(repo_names)][repo_key])"),
                comment => "Check if the repo key is known";

    reports:
        DEBUG|DEBUG_apt|DEBUG_apt_repository::
            "$(this.bundle): repository_file excluded from deletion: $(repo_exclude_files_str)";
            "$(this.bundle): Merged data repository files: $(repo_data_str)";

        !apt_import_key_cmd_exists::
            "$(this.bundle): import keys command not found or set: '$(scl.apt[import_key])'";
}

bundle agent apt_repository_key(name, key)
{
    vars:
        any::
            "filename" string => "$(def.node_template_dir)/apt/$(name).key";
            "import_key_error_codes" slist => { "1", "2", "3" };

    files:
        any::
            "$(filename)"
                comment => "Create key file name in bundle template directory",
                create => "true",
                edit_line => scl_var_to_file("$(key)"),
                edit_defaults => empty,
                classes => results("bundle", "apt_repository_key_$(key)");

       apt_repository_key_cmd_failed::
            "$(filename)"
                comment => "Import of APT Key failed so remove it",
                delete => tidy;

    commands:
        any::
            "$(scl.apt[import_key]) $(key)"
                contain => in_shell,
                classes => scl_cmd_repair ( "@(import_key_error_codes)", "apt_repository_key_cmd" ),
                action => if_elapsed("50"),
                if => or(
                    canonify("apt_repository_key_$(key)_repaired"),
                    "$(scl.apt[key_check_schedule])",
                    "APT_FORCE_KEY_CMD"
                    );

    reports:
        any::
            "$(this.bundle): Generated $(filename) with $(key)"
                if => and(
                                canonify("apt_repository_key_$(key)_repaired"),
                                "DEBUG_apt|DEBUG_apt_repository|DEBUG"
                                );

        DEBUG_apt|DEBUG_apt_repository::
            "$(this.bundle): Checking repository:$(name) with key:$(key)";
}


bundle agent apt_auth_conf()
{
    vars:
        any::
            "header" string => "## Maintained by CFengine$(const.n)#$(const.n)";
            "file" string => "$(apt.config_dir)/auth.conf";
            "mustache" string => "$(header)
{{#auth_conf}}
machine {{{machine}}}
login {{{login}}}
password {{{password}}}
{{/auth_conf}}
";

    files:
@if minimum_version(3.12)
        any::
            "$(file)"
                create => "true",
                edit_template_string => "$(mustache)",
                template_method => "inline_mustache",
                template_data => "@(scl.apt)",
                classes => if_repaired("scl$(file)");

            "$(file)"
                perms => mog("0600", "root", "root");
@endif

    reports:
        "$(this.bundle): Has been called"
            if => "DEBUG|DEBUG_apt|DEBUG_$(this.bundle)";
}

bundle agent apt_debconf()
{
    vars:
        "index" slist => getindices("scl.apt[debconf]");

    classes:
        "debconf_module_exists" expression => fileexists("$(scl.apt[debconf_module])");

    methods:
        debconf_module_exists::
            "" usebundle => apt_debconf_set(
                "$(scl.apt[debconf][$(index)][package])",
                "$(scl.apt[debconf][$(index)][field])",
                "$(scl.apt[debconf][$(index)][type])",
                "$(scl.apt[debconf][$(index)][value])");

@if minimum_version(3.11)
            "" usebundle => scl_show_data("scl", "apt[debconf][$(index)]"),
                if => "DEBGU|DEBUG_apt)|DEBUG_$(this.bundle)";
@endif
    reports:
        !debconf_module_exists::
            "$(this.bundle): Module '$(scl.apt[debconf_module])' not found, skipping debconf settings";
}

bundle agent apt_debconf_set(package, field, type, value)
{
    vars:
        "debconf_module" string => "$(scl.apt[debconf_module])";
        "return_codes" slist => { "1" };

        "class" string => canonify("$(field)"),
            comment => "$(field) will be used for setting unique classes in debconf module";

    commands:
        any::
            "$(debconf_module)"
                args   => "$(package) $(field) $(type) $(value) $(class)",
                comment => "Run $(debconf_module) $(field) $(value)",
                if => "!(APT_DEBCONF_$(class)_CHANGED|APT_DEBCONF_$(class)_NOT_CHANGED)",
                module => "true";

        !dpkg_reconfigure_succeeded::
            "DEBIAN_FRONTEND=noninteractive $(apt.dpkg_reconfigure) $(package)"
                comment => "Run $(apt.dpkg_reconfigure) $(package)",
                contain => in_shell,
                if => "APT_DEBCONF_$(class)_CHANGED",
                classes => scl_cmd_repair("@(return_codes)",  "dpkg_reconfigure");

    reports:
        any::
            "$(this.bundle): $(debconf.response)"
                if => "APT_DEBCONF_$(class)_CHANGED|APT_DEBCONF_$(class)_ERROR|APT_DEBCONF_ERROR";

        dpkg_reconfigure_failed::
            "$(this.bundle): Failed to run '$(apt.dpkg_reconfigure)' for package '$(package)'";
}

bundle agent apt_install()
{
    methods:
        any::
            "" usebundle => scl_service_packages("apt", "@(apt.packages)");
}

@if minimum_version(99.9)
# apt.cf

Source: [apt.cf](/services/apt.cf)

This bundle will be used to:
 * generate repository files in /etc/apt/sources.list.d
 * generate /etc/apt/auth.conf, via the json variable `auth_conf`
 * install apt packages
 * Copy the gpg key files if `key_file` is set for repository (prefered method)
 * Fetch the gpg key via `apt_import_key` command if `repo_key` is set for repository

The following actions are proctected by a class:
 * automatic install security update (AUTOMATIC_SECURITY_UPDATE)
 * automatic remove obsolete packages (AUTOREMOVE)
 * check the status of the package manager (CHECK_STATUS)
 * kill apt/aptitude processes that run more then 1 hour (KILL_PKG_MANAGER)
 * Check the Debian release and upgrade if needed (OS_VERSION_CHECK)
 * setting debconf values for package field(s), controlled via json data
 * disable systemctl timer services for apt, may interfere with CFEngine (APT_SYSTEMD_DISABLE)
 * remove /etc/apt/sources.list file (SOURCES_FILE_REMOVE)
 * automatic perform a dist-upgrade (DIST_UPGRADE)



The classes can be set in def.cf/json:
```json
{
"apt": {
    "classes": {
        "AUTOMATIC_SECURITY_UPDATE": "any",
        "AUTOREMOVE": "any",
        "SYSTEMD_DISABLE": "systemd"
    }
}
```

The `/etc/apt/auth.conf` is default empty. But we can configure it for password enabled repositories via `auth.conf`:
```
"auth_conf": [
    {
        "login": "readonly",
        "machine": "example.surfsara.nl/artifactory/CC-Debian-",
        "password": "secret"
    }
]
```

## Usage

The bundle can be run via:
 * `def.scl_services_enabled`
"vars": {
    "scl_services_enabled": [
            "...",
            "apt",
            "..."
    ]
}
```

The bundle will aways read the [default.json](/templates/apt/json/default.json) file
and extra json file(s) can be specified via:
 * def.cf
```
vars:
    any::
        "apt_json_files" slist => { "debconf.json" };
```

The variable must be `apt_json_files` and with this setup 1 extra json file will be merged.

### DEBUG

if you want to debug this bundle set the `DEBUG_apt` class, eg:
 * `-DDEBUG_apt`

## def.cf/json

See [default.json](/templates/apt/json/default.json) what the default values are and
which variables can be overriden.

The following must be set in host specific host file: `def.json`
```json
"vars": {
    "apt": {
        "classes": {
            "AUTOMATIC_SECURITY_UPDATE": "any"
        },
        "json_files": [ "debconf.json" ]
    }
}
```

### Debconf

Debconf is a configuration system for Debian packages. When `debconf` json data
is set. The apt bundle will configure the package with these values, eg:
```json
{
    "debconf": [
        {
            "package": "dash",
            "field": "dash/sh",
            "type": "boolean",
            "value": "false"
        },
        {
            "package": "debconf",
            "field": "debconf/frontend",
            "type": "select",
            "value": "Noninteractive"
        }
    ]
}
```

This will configure the packages `dash` and `debconf`.
@endif