zfs data pool (dpool) (#98)

* Update tailscale inventory, try out stateful-filtering flag to debug
container sidecar

* Update dpool for zfs mirror, new ts auth key

* Update flake.lock

* Add tailscale_key ansible role

* Pin versions, add pause for homeassistant_docker_compose

* Use ghcr.io for audiobookshelf_docker_image and version

* Add media folders for jellyfin

* Add dpool to extraPools in nixos config

* Add autoScrub

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Remove yamllint (ansible, don't care as long as it runs)

* Update pre-commit

* Remove pre-commit

* Initial sanoid config

* Fix typo in sanoid config

* Fix missing ; in hardware-configuration

* use bool type

* Update config to use sub-modules

* Add link to sanoid github repo

* formatting

* Add notes to README

* Update nextcloud datadir

* Update flake.lock

* Nextcloud: switch from datadir to home

* Update nextcloud to 30

* Add stop and pause steps to ansible for docker compose to not create new
tailnet node names, like books-1 when I just want books

* Update docs

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/lint.yaml

@@ -21,8 +21,8 @@ jobs:
       - name: Lint GitHub Actions
         run: actionlint
 
-      - name: Lint yaml
-        run: yamllint .
+      # - name: Lint yaml
+      #   run: yamllint .
 
       - name: Lint lua
         run: selene .

ansible/group_vars/all/vars.yml

@@ -11,48 +11,20 @@ odroid_nixos_modules:
 odroid_container_user: iancleary
 odroid_container_user_group: users
 
+data_pool: dpool
+
+# https://github.com/advplyr/audiobookshelf/pkgs/container/audiobookshelf
+audiobookshelf_docker_image: ghcr.io/advplyr/audiobookshelf
+audiobookshelf_docker_version: 2.15.1
+
 homeassistant_mosquitto_user: "{{ vault_homeassistant_mosquitto_user }}"
 homeassistant_mosquitto_password: "{{ vault_homeassistant_mosquitto_password }}"
 
 # MongoDB for Unifi, supports 3.6 through 4.4 as of 2024-02-21
+# Starting with version 8.1 of Unifi Network Application, mongodb 3.6 through 7.0 are supported.
 # https://github.com/linuxserver/docker-unifi-network-application?tab=readme-ov-file#setting-up-your-external-database
 unifi_mongo_version: 4.4
 unifi_mongo_password: "{{ vault_unifi_mongo_password }}"
+unifi_network_version: "8.3.32-ls57"
 
 iancleary_tailscale_auth_key: "{{ vault_iancleary_tailscale_auth_key }}"
-#
-#
-# Below is from old vars.yml
-#
-# iancleary_dns_user_password: "{{ vault_iancleary_dns_user_password }}"
-# iancleary_raspberrypi_user_password: "{{ vault_iancleary_raspberrypi_user_password }}"
-# pi_user_password: "{{ vault_pi_user_password }}"
-
-# github_users_authorized_keys_exclusive: true
-# github_url: https://github.com
-# github_users_absent: []
-# github_users:
-#   - name: iancleary
-#     groups: sudo,www-data
-#     # password: "{{ vault_iancleary_user_password }}"
-
-# docker_pip_version: 6.0.0
-# docker_compose_pip_version: 1.29.2
-# docker_users:
-#   - iancleary
-
-# adguard_home_user: iancleary
-# adguard_home_state: absent
-
-# unifi_controller_user: iancleary
-# unifi_controller_state: present
-
-# pi_hole_user: iancleary
-# pi_hole_state: present
-# pi_hole_web_password: "{{ vault_pi_hole_web_password }}"
-
-# pi_hole_ftlconf_local_ipv4: "{{ vault_pi_hole_ftlconf_local_ipv4 }}"
-# pi_hole_dns: "{{ vault_pi_hole_dns }}"
-# pi_hole_tz: "{{ vault_pi_hole_tz }}"
-# pi_hole_dns_entries: "{{ vault_pi_hole_dns_entries }}"
-# pi_hole_white_wild_entries: "{{ vault_pi_hole_white_wild_entries }}"

ansible/group_vars/all/vault.yml

@@ -1,62 +1,62 @@
 $ANSIBLE_VAULT;1.1;AES256
-64373835323935303631323834333861393934353838303139656532303961666237346563616365
-3666336134643131636636393839623835643237353339630a333130346235303839616138373431
-66323364653437323030393166363262653965623532643230336531643463343735653063363836
-6365616330613462620a373262303363326366386233663733376339633066633831333930303133
-61613563346637333934643532636665643961663266393237333436373863663333323335306261
-33353536653039396630346562356630316639383265656631376366383465383665353232656534
-39303233616138343630323432306530633262353039663638313065343562306638666135366338
-39323432393966646162616634316534653031633163373164373535623235383361613463303438
-35653137313161616132646265316164326661343537613032373639653934653938396539626535
-61646132333561336132366336353239306530333066613136303934346536396661613230393331
-37383634653735333634383132303161336462626661353764663030613230633230346562636636
-38336366393532616265333736663365666264303166366137616235653638323734386665386632
-65386130663863613839346537343239633636663665386336636134396638323861343865393062
-65376237353362343539303163383839643936396537313439616162646432333735663636376664
-31643033666539393233393766343632383530616538373639393633613631336666323136623761
-61633965313965373530353530303530333931643666363531366534323861336232323861313134
-32333263623835353266616536613437633931643436333137306665633462346532336238643939
-34343739653266336162336332303631636561303933626363393135343865366134396430313865
-31366431366536343432333039333632323339383364663530656134633964323632383236613439
-66643439613838636561656566343731353866656636333065363633626232383539666561353864
-66373335363531303635383034343232373933646532653636373231326130366663303539323464
-61653337363636326635663138356330306666313632393137663132646461663132373065346334
-38663463613533333635656434323032613130656663363761643030316165663031333462343865
-32353032643963356234623430363032313039643334323836633266643832336139656664623637
-37616163613233656431613233366230363739323137316238616330666462336364633036633536
-30333364633164626334386663316261353838623262303531636435376237666630373333633866
-64646433393733363033616663306533623732383331303537313764656532343739663731386437
-31346163333135343662303737343837376237323930313238633036376461393364306562306230
-34636630653566333934613866313138666662353831313638323332383837373164343033663731
-31363165346535373432633966386662356139326537383530363463613162333136643464373239
-31353637363663633762633734643737663962323766396462306361633630396432663931376139
-36316533353733323532303064353062303834343334383132343937633431323666376231633137
-37333439376263396263313932313139363735613236303936343534633339393564336634353934
-64343763373561396336313161376562336537636666616436656631393963663364323434343634
-32643235363032393263616363333862373363656662336633393837363434363565353933613063
-37366635386238336136336564336132613236346262636361306362393366353738313633656561
-64316534313432643239343336333964376262343430393536306637643232643439333062613031
-36616131303365363262313538323562373930336163303637393533386632393030373262373365
-39336363613634656332616264323465636565353434633966646163666630333732666266373263
-37393865363366306231316366346239633861663163636235613063313639623235636466346165
-64616134393733303765303966346134666532313632663930316431633633386436613363656235
-35643538346336396663303030353163356466343261636435623666343135383435356633336463
-36643936393539393830383464303736303162383564323661363362386634613963396362303031
-33336263316131353332633439393139393866323132396164333565656263326561303334356638
-66316635313465666162393034326637353766613631356332396438343037623663333036373666
-34386434383731393538393566633961306331373436336530626338306137373966336432306561
-34663335353965353030313162326638326337626333626164353930353430353864313730393638
-36623031616466636432633130363430373266656130363332353837323733656636643665653762
-63383936383037336666366533376637336439303131393937373931343239306634373064316331
-33663532393836313031663430333064336136663265616631616130323930326563653239316562
-33643133626364663733616636656637333365643332623762393135393539323963323962356437
-34386437633332613038623666646530363163633033656431613333376633323865653863396136
-32313037306537353531333732333935303230333530393838366137313666356231393463663837
-66653461323239653661653739386263303837663330383132643330326439623433326566386531
-64663836656365663364373233353231303431393161383234636330643563363566613334316638
-32373363333265643763623833366638316533373730326232633262326565663135343364343761
-38633162653862336535396663303835303635323333633834663437306164323635653836373130
-38626439333732376434346466353862303962383135343262643035336534316561666364663636
-63643635393135656363303031343836623232613338313532653334396561643437363038333834
-62316235343332336663666430643633656364633262323437326333613461633439346638316630
-3362
+36313433626537663338646338306533353332316235396161373936326230616536333431326331
+3635633638303939383837336164313532643866326131340a363634343436633435636432396235
+64653635363132343464643466353338623031666439393733633531613562653462303736336630
+6362316536393665610a363162653862336461316431346137366631343065613636336362646232
+33326337306130333739366263333563343137353831346265383766336264333838643261643436
+62666532653831623265663634386661333436386338613566656163343539343263613763663739
+35616639393031613762383535333165663363633761326630323639373431643532356334633165
+34643836313934356639663933363637623636356537643635306666306362643231633133373163
+65386664386234303638343036633238633261333239653966393438353433653233656633623935
+35376461656439653830633264613435643636313366313538306235653133393163323237663364
+66626166353336393039313338373732623833643830363163363636646664386236313365323866
+64343763666462343334386336303034353439643664306463613439376566353833316163663631
+61393737343438316331323135313762643630353231343637613064343433613262303630666662
+34643062383761393466623966656331343566376635343231353230353064306530366133346166
+62386133663162313437306166306437306264373061333134616539656563303965313662353761
+36373661613534666530313734386366646439636661663666383138306562666635633337636162
+38306535666331396662393162626566623366333635633335313739623538323735306336643631
+66326461653232333063613635383231626537663538313361373462356132353361303565373566
+64373862636165653831663961313333396233333532373962363834333730363234653565656464
+33353830656162653737656464343634626333373063346238623231363332363236623761653838
+35376264646238623630383939393164623530616335343630366164316133313666363934656162
+34376533343465343531343965366436363764346136306235386133376639323737653537373730
+65323364366563366432653036613932383235613564303339316331333738626361303066653936
+39363639653235393936386362616537343362633035633961623438333262666464643565303233
+65633163633435343134343461653133326661633633383165333862343837353836343963343561
+33613232336365633463393435663037376535376464663963636466663663396163636534336261
+30346133623261626236303563353932353762633338653038633166646165343161646634633138
+39333830313439643438646466643437343362343261656263353737363534343061386537346233
+31326162653030653838636565326466633831396635646163346539653866646663353336613831
+61393437366461653461353837323732373931353265383434356336323530636562363732303763
+38316463303239646161396562663838306334333438386431393335666630303831386637646364
+64383530396261393932393433303831396431336365336439383662363361646632636662353366
+65636530326230393633363732383332373933383336393835373733633961363036653332313963
+65333965363737376331626631316366613339333665643230313933386331356637363066663637
+30383562396439306534313165386131383638316432643363356530363265376633643864373831
+62326230363732653336373738623764633663316363666239613366376637353631336666366338
+34326237613037666663343938366634333332623332363530636537313633613734343830623838
+65326532626437303638393834313738623531373233346631393436643166373764373436306162
+37383832626639323436326431373766383861393738656233323633356166643662643135663466
+62363063383536356232396463353365346632383134363439653335653166633539383432613033
+66313062373862653165643032636666393632323236623439356533396264633038623238616230
+64643235663339353638646266393939313165366134366437656634373734353230616637393933
+66376165353531383365383737386161343832303432363136353365373130636664383330363636
+65623663383230376563343638366337623465393039613938323564346336373439323233623936
+32326431633561363539393334353035666439346635353864613066383133306461363837303939
+32333731383066313066646166353434643536393239636336366132363562306634353433623761
+37366166656534303432643430646237333930333933306165373632646339366637366565613438
+63613462353334326364333638356464393631633339663034336465376263646165363336623136
+32383564613639303938633563373539303364313865663665343163303730313030323766363933
+64623066363236363831363665663737393564333739613166336236346663633364363336636135
+61653535383231613762643334353330623461386566303037373232373134613766666130323131
+32356638656636393530336461646434653165666232653766646261353633616161376137643563
+35303436346634663363653764393330613163373931303761393832363138653034366435383735
+32356230303339353661323632313031643133346532653430356431616235373036373464633731
+66306532366132653032326335376163666130313034656531393633616633323762666637343732
+61653236376231646435356164376134376664633230333965646265623965366262336335633633
+39373239396330363162373033363965383533376139383461653135663530356461623733373735
+32393831663530326633326232346136306438343833313730666461623933333765313537316166
+64393965313334326465353464656464373432623461343730326334396239656238336264323333
+64646335396434643937323961356536376461663937616365646239653436316163653061666164
+3531

ansible/inventory

@@ -1,2 +1,2 @@
 [odroid1]
-odroid1.tail2500d.ts.net
+odroid1

ansible/playbook_odroid.yml

@@ -3,6 +3,12 @@
   hosts: odroid1
   remote_user: iancleary
   roles:
+    - {
+        role: tailscale_key,
+        become: true,
+        tags: ["tailscale_key"],
+        ansible_become_password: "{{ odroid_iancleary_user_password }}",
+      }
     - {
         role: containers_audiobookshelf,
         become: true,

ansible/roles/containers_audiobookshelf/tasks/main.yml

@@ -9,9 +9,9 @@
 
 - name: Pull audiobookshelf image
   community.docker.docker_image:
-    name: docker.io/advplyr/audiobookshelf
+    name: "{{ audiobookshelf_docker_image }}"
     source: pull
-    tag: latest
+    tag: "{{ audiobookshelf_docker_version }}"
     force_source: true
     # Select platform for pulling. If not specified, will pull whatever docker prefers.
 
@@ -20,18 +20,18 @@
     path: "{{ item }}"
     state: directory
   with_items:
-    - "/home/{{ odroid_container_user }}/Containers/audiobookshelf/tailscale-state/"
-    - "/home/{{ odroid_container_user }}/Containers/audiobookshelf/tailscale-config/"
-    - "/home/{{ odroid_container_user }}/Containers/audiobookshelf/tailscale-config/config/"
-    - "/home/{{ odroid_container_user }}/Containers/audiobookshelf/Audiobooks/"
-    - "/home/{{ odroid_container_user }}/Containers/audiobookshelf/Podcasts/"
-    - "/home/{{ odroid_container_user }}/Containers/audiobookshelf/config/"
-    - "/home/{{ odroid_container_user }}/Containers/audiobookshelf/metadata/"
+    - "/{{ data_pool }}/audiobookshelf/tailscale-state/"
+    - "/{{ data_pool }}/audiobookshelf/tailscale-config/"
+    - "/{{ data_pool }}/audiobookshelf/tailscale-config/config/"
+    - "/{{ data_pool }}/audiobookshelf/Audiobooks/"
+    - "/{{ data_pool }}/audiobookshelf/Podcasts/"
+    - "/{{ data_pool }}/audiobookshelf/config/"
+    - "/{{ data_pool }}/audiobookshelf/metadata/"
 
 - name: Copy the docker compose file to the server
   ansible.builtin.template:
     src: "audiobookshelf-docker-compose.yml"
-    dest: "/home/{{ odroid_container_user }}/Containers/audiobookshelf/docker-compose.yml"
+    dest: "/{{ data_pool }}/audiobookshelf/docker-compose.yml"
     owner: "{{ odroid_container_user }}"
     group: "{{ odroid_container_user_group }}"
     mode: "0644"
@@ -41,12 +41,21 @@
 - name: Copy the tailscale json file to the server
   ansible.builtin.template:
     src: "tailscale.json"
-    dest: "/home/{{ odroid_container_user }}/Containers/audiobookshelf/tailscale-config/config/tailscale.json"
+    dest: "/{{ data_pool }}/audiobookshelf/tailscale-config/config/tailscale.json"
     owner: "{{ odroid_container_user }}"
     group: "{{ odroid_container_user_group }}"
     mode: "0644"
 
+- name: Stop services
+  community.docker.docker_compose_v2:
+    project_src: "/{{ data_pool }}/audiobookshelf"
+    state: stopped
+
+- name: Pause for 3 seconds to allow services to fully stop
+  ansible.builtin.pause:
+    seconds: 3
+
 - name: Create and start services
   community.docker.docker_compose_v2:
-    project_src: "/home/{{ odroid_container_user }}/Containers/audiobookshelf"
+    project_src: "/{{ data_pool }}/audiobookshelf"
     state: present

ansible/roles/containers_audiobookshelf/templates/audiobookshelf-docker-compose.yml

@@ -3,23 +3,23 @@ version: "3.7"
 name: audiobookshelf
 services:
   tailscale:
-    hostname: books                         # This will become the tailscale device name
+    hostname: books # This will become the tailscale device name
     image: docker.io/tailscale/tailscale:latest
     environment:
       - TS_AUTH_KEY={{ iancleary_tailscale_auth_key }}
       - TS_SERVE_CONFIG=/config/tailscale.json
       - TS_STATE_DIR=/var/lib/tailscale
     volumes:
       - ${PWD}/tailscale-state:/var/lib/tailscale
-      - ${PWD}/tailscale-config/config:/config       # State data will be stored in this directory
-      - "/dev/net/tun:/dev/net/tun"           # Required for tailscale to work
-    cap_add:                                    # Required for tailscale to work
+      - ${PWD}/tailscale-config/config:/config # State data will be stored in this directory
+      - "/dev/net/tun:/dev/net/tun" # Required for tailscale to work
+    cap_add: # Required for tailscale to work
       - net_admin
       - sys_module
     restart: unless-stopped
 
   audiobookshelf:
-    image: ghcr.io/advplyr/audiobookshelf:latest
+    image: "{{ audiobookshelf_docker_image }}:{{ audiobookshelf_docker_version }}"
     network_mode: service:tailscale
     depends_on:
       - tailscale