feat: revive caching of Fedora qubes

- Update with cacher upstream changes; - Fix README command typos; - Restore Fedora functionality; - Update mirror list; - Move repository definitions to separate files for readability; and - Add Tailscale and Blackarch repository.
ben-grande · Jun 7, 2024 · bb38440 · bb38440
1 parent 29601d8
commit bb38440
Show file tree

Hide file tree

Showing 18 changed files with 249 additions and 95 deletions.
diff --git a/salt/sys-cacher/README.md b/salt/sys-cacher/README.md
@@ -17,9 +17,8 @@ Caching proxy server for software repositories in Qubes OS.
 ## Description
 
 The caching proxy is "sys-cacher" based on apt-cacher-ng, it stores downloaded
-packages, so that you need only download a package once for it to be used when
-updating many  The proxy is preconfigured to work out of the box
-for Debian, Ubuntu, Arch, and Fedora
+packages, so that you need only download a package once and fetch locally the
+next time you want to upgrade your system packages.
 
 When you install this package, qubes will be tagged with "updatevm-sys-cacher"
 and they will be altered to use the proxy by default. When there is "https://"
@@ -31,9 +30,11 @@ This change will be done automatically for every template that exists and is
 not Whonix based. No changes are made to Whonix templates, and updates to
 those templates will not be cached.
 
-The caching proxy supports Debian derivatives (not Whonix) and Arch Linux.
-Fedora support was dropped due to unreliability of the mirror mechanism of
-zchunk checksums when caching packages.
+The caching proxy supports:
+
+- Debian and derivatives (but not Whonix)
+- Fedora and derivatives
+- Arch Linux and derivatives
 
 ## Installation
 
@@ -136,7 +137,7 @@ By default, only templates will use the proxy to update, if you want to cache
 non-TemplateVMs updates or simply make them functional again, the qube will
 need the `service.updates-proxy-setup` feature set:
 ```sh
-qvm-tags add QUBE updatevm-sys-cacher
+qvm-tags QUBE add updatevm-sys-cacher
 qvm-features QUBE service.updates-proxy-setup 1
 sudo qubesctl --skip-dom0 --targets=QUBE state.apply sys-cacher.install-client
 ```
@@ -145,13 +146,13 @@ Don't forget to restart the qube.
 If you don't want or can't restart the qube, such as DispVMs, where you would
 lose the current session:
 ```sh
-qvm-tags add QUBE updatevm-sys-cacher
+qvm-tags QUBE add updatevm-sys-cacher
 qvm-features QUBE service.updates-proxy-setup 1
+sudo qubesctl --skip-dom0 --targets=QUBE state.apply sys-cacher.install-client
 qvm-run --user=root QUBE -- "
 touch /var/run/qubes-service/updates-proxy-setup
 /usr/bin/apt-cacher-ng-repo
 systemctl restart qubes-updates-proxy-forwarder.socket"
-sudo qubesctl --skip-dom0 --targets=QUBE state.apply sys-cacher.install-client
 ```
 
 ## Uninstallation
@@ -175,7 +176,7 @@ If you want to use the standard proxy for a few qubes, only uninstall it
 from the templates that you don't want to cache packages:
 ```sh
 sudo qubesctl --skip-dom0 --targets=QUBE state.apply sys-cacher.uninstall-client
-qvm-tags del QUBE updatevm-sys-cacher
+qvm-tags QUBE del updatevm-sys-cacher
 ```
 
 If you tagged manually a qube that is unsupported, updates for that qube will

diff --git a/salt/sys-cacher/configure.sls b/salt/sys-cacher/configure.sls
@@ -9,15 +9,6 @@ SPDX-License-Identifier: AGPL-3.0-or-later
 include:
   - dotfiles.copy-x11
 
-"{{ slsdotpath }}-install-rc.local":
-  file.managed:
-    - name: /rw/config/rc.local.d/50-sys-cacher.rc
-    - source: salt://{{ slsdotpath }}/files/server/rc.local.d/50-sys-cacher.rc
-    - mode: '0755'
-    - user: root
-    - group: root
-    - makedirs: True
-
 "{{ slsdotpath }}-install-qubes-firewall":
   file.managed:
     - name: /rw/config/qubes-firewall.d/50-sys-cacher

diff --git a/salt/sys-cacher/create.sls b/salt/sys-cacher/create.sls
@@ -56,6 +56,7 @@ features:
 - enable:
   - servicevm
   - service.crond
+  - service.apt-cacher-ng
 - disable:
   - service.cups
   - service.cups-browsed

diff --git a/salt/sys-cacher/files/admin/list-extra-tag.sh b/salt/sys-cacher/files/admin/list-extra-tag.sh
@@ -17,7 +17,10 @@ wanted=""
 for qube in ${tagged}; do
   get_os_distro "${qube}"
   case "${os_distro}" in
-    debian|ubuntu|linuxmint|kali|kicksecure|arch)
+    debian|kali|kicksecure|parrot| \
+    ubuntu|linuxmint| \
+    arch|blackarch| \
+    fedora)
       continue
       ;;
     "")
@@ -43,7 +46,10 @@ for qube in ${tagged}; do
           esac
           get_os_distro "${template}"
           case "${os_distro}" in
-            debian|ubuntu|linuxmint|kali|kicksecure|arch)
+            debian|kali|kicksecure|parrot| \
+            ubuntu|linuxmint| \
+            arch|blackarch| \
+            fedora)
               continue
               ;;
             ## Qube is not supported.

diff --git a/salt/sys-cacher/files/admin/tag.sh b/salt/sys-cacher/files/admin/tag.sh
@@ -18,7 +18,10 @@ wanted=""
 for qube in ${templates}; do
   os_distro="$(qvm-features "${qube}" os-distribution || true)"
   case "${os_distro}" in
-    debian|ubuntu|linuxmint|kali|arch)
+    debian|kali|kicksecure|parrot| \
+    ubuntu|linuxmint| \
+    arch|blackarch| \
+    fedora)
       wanted="${wanted:+"${wanted} "}${qube}"
       ;;
     *) continue

diff --git a/salt/sys-cacher/files/client/bin/apt-cacher-ng-repo b/salt/sys-cacher/files/client/bin/apt-cacher-ng-repo
@@ -94,9 +94,6 @@ reject_os(){
 set_proxy_os(){
   if test -e /etc/fedora-release; then
     ## Fedora
-    ## Uninstall because it leads to many zchunk checksum mismatch problems.
-    action="uninstall"
-    echo "${0##*/} doesn't work well on Fedora, uninstalling." >&2
 
     if test -w /etc/dnf/dnf.conf; then
       set_proxy_marker /etc/dnf/dnf.conf "zchunk=False

diff --git a/salt/sys-cacher/files/server/conf/acng.conf b/salt/sys-cacher/files/server/conf/acng.conf
@@ -71,21 +71,21 @@ Port:8082
 # Repository remapping. See manual for details.
 # In this example, some backends files might be generated during package
 # installation using information collected on the system.
-# Examples:
-#Remap-debrep: file:deb_mirror*.gz /debian ; file:backends_debian # Debian Archives
-Remap-alxrep: file:archlx_mirrors /archlinux # ; file:backend_archlx # Arch Linux
-Remap-alxrep: file:archlx_mirrors /archlinux # ; file:backend_archlx # Arch Linux
+
+# Default repositories
+Remap-alxrep: file:archlx_mirrors file:archlx_mirrors_extra /archlinux # ; file:backend_archlx # Arch Linux
+Remap-blackarch: file:blackarch_mirrors_extra /blackarch
 Remap-cygwin: file:cygwin_mirrors /cygwin # ; file:backends_cygwin # incomplete, please create this file or specify preferred mirrors here
-Remap-debrep: https://deb.debian.org http://deb.debian.org  file:deb_mirrors.gz /debian
+Remap-debrep: file:debian_mirrors_extra  file:deb_mirrors.gz /debian ; file:backends_debian
 Remap-epel:   file:epel_mirrors # Fedora EPEL
-Remap-fedora: file:fedora_mirrors # Fedora Linux
-Remap-fedora: file:fedora_mirrors # Fedora Linux
+Remap-fedrep: file:fedora_mirrors file:fedora_mirrors_extra ; https://mirrors.kernel.org/fedora/ https://ftp-stud.hs-esslingen.de/pub/fedora/linux/  # Fedora Linux
 Remap-gentoo: file:gentoo_mirrors.gz /gentoo ; file:backends_gentoo # Gentoo Archives
 Remap-klxrep: file:kali_mirrors /kali ; file:backends_kali # Kali Linux Archives
-Remap-secdeb: security.debian.org security.debian.org/debian-security deb.debian.org/debian-security /debian-security cdn-fastly.deb.debian.org/debian-security ; deb.debian.org/debian-security security.debian.org cdn-fastly.deb.debian.org/debian-security
+Remap-secdeb: file:debiansec_mirrors_extra /debian-security
 Remap-sfnet:  file:sfnet_mirrors # ; file:backends_sfnet # incomplete, please create this file or specify preferred mirrors here
 Remap-slrep:  file:sl_mirrors # Scientific Linux
 Remap-uburep: file:ubuntu_mirrors /ubuntu ; file:backends_ubuntu # Ubuntu Archives
+
 # Qusal external repositories
 Remap-dockerrep: https://download.docker.com http://download.docker.com
 Remap-elementrep: https://packages.element.io http://packages.element.io
@@ -99,6 +99,7 @@ Remap-qubesdebrep: https://deb.qubes-os.org http://deb.qubes-os.org
 Remap-qubesyumrep: https://yum.qubes-os.org http://yum.qubes-os.org
 Remap-signalrep: https://updates.signal.org http://updates.signal.org
 Remap-syncthingrep: https://apt.syncthing.net http://apt.syncthing.net
+Remap-tailscalerep: https://pkgs.tailscale.com http://pkgs.tailscale.com
 Remap-whonixdebrep: https://deb.whonix.org http://deb.whonix.org
 
 # Virtual page accessible in a web browser to see statistics and status
@@ -271,8 +272,9 @@ FollowIndexFileRemoval: 1
 #
 # To see examples of the expected syntax, run: apt-cacher-ng -p debug=1
 #
-PfilePatternEx: .*yaml.gz$|.*fedora.*arch=x86_64$|.*f[0-9]+&arch=x86_64
-VfilePatternEx: .*fedora.*updateinfo.*xml.zck$|^/\?release=[0-9]+&arch=.*|.*/RPM-GPG-KEY.*|.*\?repo=fedora|.*pkg.tar.zst.sig|.*archlinux.*sha256sums.txt|.*archlinux/iso.*tar.gz.sig
+PFilePatternEx: .*yaml.gz$|.*fedora.*arch=x86_64$|.*f[0-9]+&arch=x86_64
+VFilePatternEx: .*fedora.*updateinfo.*xml.zck$|^/\?release=[0-9]+&arch=.*|.*/RPM-GPG-KEY.*|.*\?repo=fedora|.*archlinux.*sha256sums.txt|.*archlinux/iso.*tar.gz.sig|.*arch*.db|.*arch*.db.sig
+
 # SPfilePatternEx:
 # SVfilePatternEx:
 # WfilePatternEx:
@@ -360,7 +362,7 @@ VfilePatternEx: .*fedora.*updateinfo.*xml.zck$|^/\?release=[0-9]+&arch=.*|.*/RPM
 # details.
 #
 # Example:
-DontCache: .*fedora.*updates.*updateinfo.xml.zck .*fedora.*updates.*repomd.xml
+# DontCache: .*.local.university.int
 
 # Default permission set of freshly created files and directories, as octal
 # numbers (see chmod(1) for details).
@@ -418,6 +420,7 @@ LocalDirs: acng-doc /usr/share/doc/apt-cacher-ng
 # the safety period is over.
 #
 # KeepExtraVersions: 0
+KeepExtraVersions: 1
 
 # Optionally uses TCP access control provided by libwrap, see hosts_access(5)
 # for details. Daemon name is apt-cacher-ng.

diff --git a/...acher/files/server/mirrors/archlx_mirrors → ...er/files/server/conf/archlx_mirrors_extra b/...acher/files/server/mirrors/archlx_mirrors → ...er/files/server/conf/archlx_mirrors_extra
diff --git a/salt/sys-cacher/files/server/conf/blackarch_mirrors_extra b/salt/sys-cacher/files/server/conf/blackarch_mirrors_extra
@@ -0,0 +1,159 @@
+# SPDX-FileCopyrightText: 2024 unman <[email protected]>
+# SPDX-FileCopyrightText: 2024 Benjamin Grande M. S. <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+http://au.mirrors.cicku.me/blackarch/
+https://au.mirrors.cicku.me/blackarch/
+http://blackarch.mirror.digitalpacific.com.au/
+rsync://mirror.digitalpacific.com.au/blackarch/
+
+http://mirror.easyname.at/blackarch/
+ftp://mirror.easyname.at/blackarch/
+rsync://mirror.easyname.at/blackarch/
+
+http://ca.mirrors.cicku.me/blackarch/
+https://ca.mirrors.cicku.me/blackarch/
+
+https://mirrors.hust.edu.cn/blackarch/
+https://mirrors.nju.edu.cn/blackarch/
+https://mirror.sjtu.edu.cn/blackarch/
+https://mirrors.tuna.tsinghua.edu.cn/blackarch/
+https://mirrors.ustc.edu.cn/blackarch/
+https://mirrors.aliyun.com/blackarch/
+http://mirrors.aliyun.com/blackarch/
+
+http://mirrors.dotsrc.org/blackarch/
+ftp://mirrors.dotsrc.org/blackarch/
+
+http://mirror.uta.edu.ec/blackarch/
+ftp://mirror.uta.edu.ec/blackarch/
+rsync://mirror.uta.edu.ec/blackarch/
+http://mirror.cedia.org.ec/blackarch/
+
+http://blackarch.leneveu.fr/blackarch/
+http://blackarch.pi3rrot.net/blackarch/
+http://mirror.cyberbits.eu/blackarch/
+https://mirror.cyberbits.eu/blackarch/
+rsync://rsync.cyberbits.eu/blackarch/
+
+https://www.blackarch.org/blackarch/blackarch/
+rsync://blackarch.org/blackarch/
+http://de.mirrors.cicku.me/blackarch/
+https://de.mirrors.cicku.me/blackarch/
+https://mirrors.dr460nf1r3.org/repos/blackarch/
+http://ftp.halifax.rwth-aachen.de/blackarch/
+https://ftp.halifax.rwth-aachen.de/blackarch/
+ftp://ftp.halifax.rwth-aachen.de/blackarch/
+rsync://ftp.halifax.rwth-aachen.de/blackarch/
+http://blackarch.unixpeople.org/
+https://blackarch.unixpeople.org/
+rsync://blackarch.unixpeople.org/blackarch/
+http://mirror.undisclose.de/blackarch/
+https://mirror.undisclose.de/blackarch//
+rsync://mirror.undisclose.de/blackarch/
+
+http://ftp.cc.uoc.gr/mirrors/linux/blackarch/
+ftp://ftp.cc.uoc.gr/mirrors/linux/blackarch/
+rsync://[email protected]/blackarch
+
+http://mirrors.cicku.me/blackarch/
+https://mirrors.cicku.me/blackarch/
+http://www.mirrorservice.org/sites/blackarch.org/blackarch/
+rsync://rsync.mirrorservice.org/blackarch.org/blackarch/
+http://mirrors.gethosted.online/blackarch/blackarch/
+https://mirrors.gethosted.online/blackarch/blackarch/
+https://uk.mirrors.fossho.st/blackarch//os/
+
+http://quantum-mirror.hu/mirrors/pub/blackarch/
+https://quantum-mirror.hu/mirrors/pub/blackarch/
+rsync://quantum-mirror.hu/blackarch
+
+http://in.mirrors.cicku.me/blackarch/
+https://in.mirrors.cicku.me/blackarch/
+https://mirror.albony.xyz/blackarch/
+
+http://mirror.blackrepo.com/ https://mirror.blackrepo.com/
+
+http://blackarch.mirror.garr.it/mirrors/blackarch/
+rsync://blackarch.mirror.garr.it/blackarch/
+
+http://jp.mirrors.cicku.me/blackarch/
+https://jp.mirrors.cicku.me/blackarch/
+http://www.ftp.ne.jp/Linux/packages/blackarch/
+http://ftp.kddilabs.jp/Linux/packages/blackarch/
+https://ftp.kddilabs.jp/Linux/packages/blackarch/
+
+http://kr.mirrors.cicku.me/blackarch/
+https://kr.mirrors.cicku.me/blackarch/
+
+http://md.mirrors.hacktegic.com/blackarch/
+https://md.mirrors.hacktegic.com/blackarch/
+rsync://md.mirrors.hacktegic.com/blackarch/
+
+http://mirror.serverion.com/blackarch/
+https://mirror.serverion.com/blackarch/
+ftp://mirror.serverion.com/blackarch/
+rsync://mirror.serverion.com/opnsense
+http://mirror.neostrada.nl/blackarch/
+https://mirror.neostrada.nl/blackarch/
+ftp://mirror.neostrada.nl/blackarch/
+rsync://mirror.neostrada.nl/blackarch/
+
+http://nz-mirror.intergrid.com.au/blackarch/
+ftp://nz-mirror.intergrid.com.au/blackarch
+
+http://ftp.icm.edu.pl/pub/Linux/dist/blackarch/
+ftp://ftp.icm.edu.pl/pub/Linux/dist/blackarch/
+rsync://ftp.icm.edu.pl/pub/Linux/dist/blackarch/
+gopher://ftp.icm.edu.pl/1/pub/Linux/dist/blackarch/
+
+http://eu.mirrors.cicku.me/blackarch/
+https://eu.mirrors.cicku.me/blackarch/
+
+https://repository.su/blackarch/
+rsync://repository.su/blackarch/
+http://mirror.truenetwork.ru/blackarch/
+ftp://mirror.truenetwork.ru/blackarch/
+rsync://mirror.truenetwork.ru/blackarch/
+http://mirror.yandex.ru/mirrors/blackarch/
+ftp://mirror.yandex.ru/mirrors/blackarch/
+rsync://mirror.yandex.ru/mirrors/blackarch/
+
+http://sg.mirrors.cicku.me/blackarch/
+https://sg.mirrors.cicku.me/blackarch/
+http://download.nus.edu.sg/mirror/blackarch/
+https://download.nus.edu.sg/mirror/blackarch/
+
+http://mirror.zetup.net/blackarch/
+
+http://mirror.easyname.ch/blackarch/
+ftp://mirror.easyname.ch/blackarch/
+rsync://mirror.easyname.ch/blackarch/
+https://mirror.tillo.ch/ftp/blackarch/
+http://mirror.tillo.ch/ftp/blackarch/
+ftpes://mirror.tillo.ch/blackarch/
+ftp://mirror.tillo.ch/blackarch/
+rsync://mirror.tillo.ch/blackarch/
+
+http://ftp.linux.org.tr/blackarch/
+ftp://ftp.linux.org.tr/blackarch/
+rsync://rsync.linux.org.tr/blackarch/
+
+http://mirror.archlinux.tw/BlackArch/
+https://mirror.archlinux.tw/BlackArch/
+http://blackarch.cs.nycu.edu.tw/
+https://blackarch.cs.nycu.edu.tw/
+rsync://blackarch.cs.nycu.edu.tw/blackarch/
+
+http://blackarch.pr0s3c.nl/blackarch/
+https://us.mirrors.fossho.st/blackarch/
+https://blackarch.pr0s3c.nl/blackarch/
+http://mirror.math.princeton.edu/pub/blackarch/
+https://mirror.math.princeton.edu/pub/blackarch/
+rsync://mirror.math.princeton.edu/pub/blackarch/
+http://distro.ibiblio.org/blackarch/
+ftp://distro.ibiblio.org/blackarch/
+https://mirror.team-cymru.com/blackarch/
+ftp://mirror.team-cymru.com/blackarch/
+rsync://mirror.team-cymru.com/blackarch/
diff --git a/salt/sys-cacher/files/server/conf/debian_mirrors_extra b/salt/sys-cacher/files/server/conf/debian_mirrors_extra
@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2024 unman <[email protected]>
+# SPDX-FileCopyrightText: 2024 Benjamin Grande M. S. <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+http://deb.debian.org/debian
+
+https://deb.debian.org/debian
diff --git a/salt/sys-cacher/files/server/conf/debiansec_mirrors_extra b/salt/sys-cacher/files/server/conf/debiansec_mirrors_extra
@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2024 Benjamin Grande M. S. <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+http://cdn-fastly.deb.debian.org/debian-security
+http://deb.debian.org/debian-security
+http://security.debian.org
+http://security.debian.org/debian-security
+
+https://cdn-fastly.deb.debian.org/debian-security
+https://deb.debian.org/debian-security
+https://security.debian.org
+https://security.debian.org/debian-security