use the same signing key for all clouds on Zuul, use but one Zuul job (…

…#699)

* use the same signing key for all clouds on Zuul, use but one Zuul job
* Fix missing quotation marks in yaml
* use scs-test-runner to simplify Zuul job
* add explicit config because of subdirectory
* Use debugging feature of cleanup.py (also make it more secure)
* adapt bootstrap.yaml
* fix: add missing region-a to config.toml
* Fix: protect secrets better with no_log

---------

Signed-off-by: Matthias Büchse <[email protected]>
Co-authored-by: Martin Morgenstern <[email protected]>
Co-authored-by: Artem Goncharov <[email protected]>

.zuul.d/config.yaml

@@ -5,17 +5,7 @@
     merge-mode: "squash-merge"
     periodic-daily:
       jobs:
-        - scs-check-artcodix
-        - scs-check-pco-prod1
-        - scs-check-pco-prod2
-        - scs-check-pco-prod3
-        - scs-check-pco-prod4
-        - scs-check-poc-kdo
-        - scs-check-poc-wgcloud
-        - scs-check-regio-a
-        - scs-check-syseleven-dus2
-        - scs-check-syseleven-ham1
-        - scs-check-wavestack
+        - scs-check-all
     periodic-hourly:
       jobs:
         - scs-check-gx-scs-main
@@ -37,8 +27,7 @@
      - name: clouds_conf
        secret: SECRET_STANDARDS
     vars:
-      cloud: gx-scs
-      secret_key: gx_scs_key
+      preset: default
     pre-run:
      - playbooks/pre.yaml
      - playbooks/pre_cloud.yaml
@@ -48,68 +37,7 @@
     parent: scs-check-gx-scs
     branches: main
 - job:
-    name: scs-check-artcodix
+    name: scs-check-all
     parent: scs-check-gx-scs-main
     vars:
-      cloud: artcodix
-      secret_key: cnds_key
-- job:
-    name: scs-check-pco-prod4
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: pco-prod4
-      secret_key: pco_prod4_key
-- job:
-    name: scs-check-pco-prod3
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: pco-prod3
-      secret_key: pco_prod3_key
-- job:
-    name: scs-check-pco-prod2
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: pco-prod2
-      secret_key: pco_prod2_key
-- job:
-    name: scs-check-pco-prod1
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: pco-prod1
-      secret_key: pco_prod1_key
-- job:
-    name: scs-check-poc-kdo
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: poc-kdo
-      secret_key: poc_kdo_key
-- job:
-    name: scs-check-poc-wgcloud
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: poc-wgcloud
-      secret_key: poc_wgcloud_key
-- job:
-    name: scs-check-regio-a
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: regio-a
-      secret_key: regio_a_key
-- job:
-    name: scs-check-syseleven-dus2
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: syseleven-dus2
-      secret_key: syseleven_dus2_key
-- job:
-    name: scs-check-syseleven-ham1
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: syseleven-ham1
-      secret_key: syseleven_ham1_key
-- job:
-    name: scs-check-wavestack
-    parent: scs-check-gx-scs-main
-    vars:
-      cloud: wavestack
-      secret_key: wavestack_key
+      preset: all

.zuul.d/secure.yaml

@@ -24,7 +24,7 @@
           uHYP40Lq1Rd4LRQjT3TW9TybgM2Jvt6sd7sCKM7KQ4/fGyj8BswRThBNzuJMN83QmmAiq
           phthkp6X2A7ELMd22wjrOy7ruwZfXObhagJNis4x/t55fdDpnZcW3KeeqOJAv2xkD4StF
           32RSKLdIRbtWLsouOYPNloVFwPAykbrFkDfH2lyy0LJS9gWyK7t6u4Ks3P7hUE=
-      gx_scs_key: !encrypted/pkcs1-oaep
+      zuul_ci_signing_key: !encrypted/pkcs1-oaep
         - gbtzWcQo4LytBGfTskgeFs0bFXzAZo2R33ljlNFAfNzdzlrPDnrEljyys+Bp9+yjEfcG/
           rq8YeW0FVVYYulnmkasfgbUP8lMmsMHli5AwCLB00QjqCsy6Ixc1ELUr+KTTSYXyU8qhT
           d0HoofhFvMoU8nb34KvJ18LbyU1huLHZSs+3kbZQeK85rV7BMCmIwJUnGxZrxmI8tCvrT
@@ -57,17 +57,6 @@
           Vjo5fBpIjEOLNtA40vx0Agx1Red2eMFA5oLcxiDMBSP4nLiI1QNROSnGc7GYoIT5kyq8k
           P5Uw261H4un2xvpO9ZBuHEV5mskTiBazTPDwaqU3yZoyGIvdQl2Uo6S3Glnf41YH+k/hN
           jxzziQB9sT2glnmmzrzZ22xPZg+sTngWS/3svRYV9BXKnMgq8jjR6eIKGYSwyQ=
-      cnds_key: !encrypted/pkcs1-oaep
-        - vahPB8skzx04tg37UH+C/ilXjpQy9RPGf67ED/5ahAPsW8syyRLcrojgiecL7VIuRSsnZ
-          IhwqM6ofwlkCBHUgQtN2hRZUH3iDW5kDj7yc5tqPCpr16Q/XL+7yDlAgxLzHsifTiqip8
-          YNxAyNKi5ToyRl8RLZnjSFW5vvbGz3omv1VJsT9O2XZ21WNAMSjXkjtUietYznw/Z2mJE
-          1qP1zFTQh+0F/tj877vJINEC6wLX6LThydT8zsLM1MB48fLYHPvaXMNDFXVHLUn9RxYTC
-          g1OwgBkuANPnl7h6b7mvjHzF09RPAdpel15qJhrFPagEGhUihIHLH9d44EnA9PJuLybo/
-          ksZ3DukUOSMzUM3pxmw/TJZSztjLS9MIoKMg7Ct1o17f50oJeZWPIUSO6I2gjIIpR46R5
-          sINXRRNHkRJxOOI40IAkS6twYoOGtiTsNjJc8R9Eb146bWM5Vm0QEsAf5MGCvAZzDAlAF
-          5+JhrBTwV3GOu9ZIdXttMLhbQnJAOySYUf3bVzCjylNSddN+4s/3P9cbU2FCz7mOxvzfV
-          Xtu9CWvc9L+VFi6D7vASwETIi4bwCYXFJEDEPcjgvY1NBJNR0jtKid5oSHLGpDtrk6ZQD
-          APFVAuuDE1czte4g4cwAU1U0gialh1so6nhG9HjFM3cHnA9WjlHdg8PtJE9vuY=
       cnds_ac_id: !encrypted/pkcs1-oaep
         - apArEei+9Eu0NrelesebDZwUjGvbep58dAQowtcCEn1hzbUAqMxWm74Ibk1e+C3prH5Zv
           tcj+okUoyfelP8ExqCZL55cOZVentoyg/VfiwZlR1z9N/p3vDkN3sx8z6o7CJux2oFMkg
@@ -90,17 +79,6 @@
           o9slrGAyL7g3zdjNlUJA04U33SNCvaCxL8fac6JZ15vrqeW4g4AB4+rx7fYKAnOVg2FOL
           5jMOKsiGgfLvz1KZ9c6Q1ThfeCQzG9waWJnyCx2R2tEtyQ17hIW6Rzo1RzmQkUyvLN9TJ
           CLSZCUoR+2Ut+ZlpDi3vVushWWLXyjj8ojblTO/zqlbQ1A+7d/C+5x2mrh2T/0=
-      poc_kdo_key: !encrypted/pkcs1-oaep
-        - GaEAEsKrGB0/vq5XkFBqO7nUMguwC19s795gUJHeha1Bp4Mua8FgJRZgLHSZBbqp1Uyba
-          heKf5soKgw/82jLsMMcPenkigIpuTgtVhmgnLkcp4el4LaGcyxpXbUl4nsVrxWY2+wkiP
-          7wrIaZ/THNERQ5bqafMh1Wd39FnX/O3tDhwYLhONViP9SY8JxbRYOiQmzC0hL7/mMhrUC
-          NB3GDV+SJ1/W1AJy6MfEmdfVIwQWH5hFWLQMNvQ8wN2MBRpV25C22gu18GaLvoz2OhJx8
-          Rx3EdiyxfB85pAhYkd/pzwGSL4wcEhsXNAgY9blWJEQuc/BKjocyT6zBeiTzkDmOpT/5a
-          QTKB890kqCL3CerYestYtcNikFVdAgaK1KcDKg1lm9b/hOU8jP/nnogI2DAS/6LzzOeN7
-          TpMiuZDLN2VE/N9bzCVuzLh68BCwCRYt4N9QJPSy7fygKI2tbCRI4d6hGV5ukeWZ9ERzo
-          SCkWOnK4rz4pMw9C+WyxKYEKK9b3GEmMeUHO+zT9wJRvnEDClI8McrxCqOtozD1VdMpbM
-          E0i7Llh0b8u0uQ/5cNDDBQUVxH5gXwsmbtaXicgkdfebAUo2lhL5EzuVr39YgRc+zT+ZG
-          2oqOZQe3c52+rxHzHDyqLz5+vphOvf8rrMapqzEhTTQcM4jpG2hwiXfu+0Hda4=
       poc_kdo_ac_id: !encrypted/pkcs1-oaep
         - l/2ts4lmTN611PMcpMhpNrYqoiiuoZkqUTwPKfLwAEYLF68JBCuCy2iknr8fjz9FP+103
           vxqM6wq/el7kIGPUBqpb33DBJ54nS3uLUrA57vB2YoJVa99w0x/3NDsXGuC7EpJnHnkl+
@@ -123,17 +101,6 @@
           NIadxXBGLDDqTWaiOp0Mg9eXQ7tz+8lOg23uojzyXzgNiDGu77azuNkfJ4iosBHUraYEE
           cewhUsF1mXh3le03eEUnMVnnRBmFTsAyygjhLtUwnDDSoJHp3g1e1dRTd/nihsOjSLwBa
           vNFgP3N8s1smIbNMpjyMjQiTbXQ87Sqx6ZmUoSX+1tKu8tJU0k04y00xaFYWxE=
-      poc_wgcloud_key: !encrypted/pkcs1-oaep
-        - jxVFFxbR/UqXioWLk+RUWCQmrocQVVDQ370fOdEhzj8KDdXLY9wCxBXNi03niZnt++t4V
-          FudgJ7wEZgajv2s98N0ZRiY5GmJKppMxKPva7A7qbMCuu2wSXJEf+Pj9pVs6QVKsZho2d
-          wmlgaC+QwWtpzGP7lbgS7cv6lRtsrselLvusyMLgldCZDVvwmUxvRTaO++u4S76sDyz8W
-          yDDgmXMJo2KVDRReeIxwfYDzGkvDZlnse6Zh/6mPDq6Gr55+bYQWA1lF2B47tBoy9+JWd
-          17f1dv+1HYCaASBmTJZWYhK+8rjreuxutQTXn4Wh9pDuJwiP0NXzjguL0aIe1IZS4Y7+M
-          tT/WTWLR7FfWTW0A0aAKfGF6nVAJF14AtQncAfD51VUbnLHzTPQQkNFkpNLAXI/C9jXuw
-          +Mdtkx4VbD2T+zUB7raxkY3T5/IGpj0lnashCpHUzMPVsAwfjmnB3qCzoY+YhRTxvaWEI
-          rPrUG6tuxIgoORf9VqXJkm6oMB84caLtYiVxOogmDWgVWRegBvGt5nu02B1Bd38TF0OMx
-          S356QOUughSu7zedQpKhEYSphuYE3NIYB+Qibb77Gd5WF1DgURfZux/8K3n/QII7s2sX4
-          9PnGnYmRczEUUQ5ZGmXW6wF8UTJLN69f7ibXh+kuqFipx49nBtYN1Ubpf/vrL8=
       poc_wgcloud_ac_id: !encrypted/pkcs1-oaep
         - dQIs3NJt1CpP1925+b9QjjwonqjmiuCl1ewxw160yIEHQ/qyQiwutJbsg4IYS9XKhKc2X
           GumOOpLY7+/uNRR5pZmEfOdlGnPoJvVhYtCqHBFy7xQ6NLHKFxCT8zHM9ppSl1Hjc2G2F
@@ -156,17 +123,6 @@
           uZq8xRkPwlISeLi1SN23QVtBz2JIIjxa/TydCnpwAN8XEWBLQJYcqDomD+jldjf7CDIgS
           cZ/KVePBCnPpnAU8LFTR3JYC/t4N+ZY0m+rWLsQ6XUyG0lhV0uO4U9dxD/3C2F2W/U/gG
           KMDAb8fySxCweaeCWncCLPSZcLB1A7OiNGNhNwSCYtTjbHtUysssQi8CbT4m3c=
-      pco_prod1_key: !encrypted/pkcs1-oaep
-        - TvtIkva7HC5wduW4hYaBsqIqZKQ0C036wxPsJfnAHgsh7GOqVL53ZCYaHuca+APnSg3Of
-          X/wR7KmsGi1fSxgMbVx1LXwKJHItoVMPOdeP7B9skxgMYgvm/55+vEuaBCcQQrlhRoGVI
-          fqhEgRODmRy5DqF8DXcCIxirbhN9XsJbvbqmJBO/uyFD4d1zOc/ZVZzIGPcFhlLc5Y9Y1
-          96DeXvql6WIOaXZcU7DkBoLnD7m8ZxMzVsz3mvst1v+6PfY2A3ZB3Td2GnpUjDxJYUewN
-          WkqFnf2i1Ims6afT/qitscReatiVCxCy/A6UMHYsLnureucq7+g3jf1pRG+rLRv83+DVP
-          iF/CUTCBO58hj9P/TILcmG9W1W1hrgPrDMntjCgw5Kj3SDqSqDS10svClyM0B3dtXLSRQ
-          QLzCMKfQ3qqr1VoXMsRxmCaCwEpZ/3tqdVg6qN0XPkxlknDX64GqM6YMFjysEPKJjbWhd
-          AKkrMg2xokz6PTmwV6mDS4EAQlYx0QJ/MzSjYdkHj+rKLmefrob2W8M4qp5OduWEwdB0X
-          QEZESWSuNWUEUEWjK5LvLZ2g8K+oKK/JpVPgiL7xgb9krdR/CS+VwTBbhBxko5JTTEl0V
-          q3MrikIA4QF2/5cbNyG4OZ1tV/5jKrliQ80yqpKkiH6eMddkzg9axewNlcIuAo=
       pco_prod1_ac_id: !encrypted/pkcs1-oaep
         - jGScb1B/BfnuDdDnfsJoHnVRaeiTAX1fCB3eYBuUx6grQTQ2SorKWeUeVWqznfJJF0Pug
           uE09n6oCwZE3hxzI2VxFA+o4wDBA3azasAs8N3vV+QyFYF5dl+5K1M0xwdkhqAyefw5n8
@@ -189,17 +145,6 @@
           VG+WGHBasBsBieTos/OujXFVqMDgiiyB5G6crr17AKMRldxAcgvPSTG279hdstNCNClQ9
           xE2Yf0v8vXJzM/AWOMJgFanBZ35QDPIYMH6+aMh+BXj/Q0zYpWZ+y1vg358y8OYX4Q+NM
           3AGCgXvUVG3wblRSlUWUac7enMNYxlvHXTeKNLYXwut+A1qPuYFWwE+EvjReEM=
-      pco_prod2_key: !encrypted/pkcs1-oaep
-        - b41Gba5izbyr1t8GxYLkLYsW9eM1eW+5XDX4Nm4nMLqXb30DZximCKQd5MRJQCYGcJ5D2
-          /NPyp1JNXdsuAK7/kkfp6RAaXVI8RkuPwIhMWCBmdO3vTHk9bdomFb/YwqraZTL8Bqdcv
-          Em2l0/V6VNEjwqv1KyjT6mr8Sr7wUe84v/mvKxAxPcI6CO5yQEh7YgA8peOvSM/p6Uqzp
-          JvAjqRsI1NzM5i5rzW+NlqWJt1w95H+LHxnRCZRTl9Pp+wWiYiGvKFJ+ZUYlGtlVth9/d
-          IcY9d9rIhZib9vI6fBqP92kP2Wm5JArhP9nLMFChkK3NaaMBjIDSgXoLP0CpXP7rbsJmG
-          AbQtqwB7qXKQyY8S1jFBo+MQ+IkE7eFl+elPnr47QvUpOIkaQnXUCexr70/PhX24VAFIE
-          TvbisNpNmUPuB2tG/zMnATvb+EwOKy+OK6wOKofTX2IDxTnfpq0UVr0ii8ZoIbA8RsKDG
-          qwtZ4ujg3ntlrLIwN/n+sxDPeenT4DpI4W86ukRrp5xcqEcrwqX4Dh97SzeIBae0bngL0
-          wgQmKK36VPSIXBNT2k8vQebvbrygzJzOpbjKLTGX/NG9Wb6fzdjckizdGs2nupQmEmYa0
-          rx+ziMVOUfKRImUG/DkKpHxHvqI5ldfd3D5dxBya+Uzza1VGcNyXI71rRI9224=
       pco_prod2_ac_id: !encrypted/pkcs1-oaep
         - cjuaL7RFh1f4LPVP5nKv4eEOOIPKFVsK8vXtv390lX/MRDLxJGA/lb8xxRo77tKFOPlUA
           65jEHYyeDMDn1RP+8Fqq6lEQGo9X3986zNj9+NlaINv/pRHfcNCMltEoLVsOTuBeg+DJC
@@ -222,17 +167,6 @@
           FokGXgiJvYxZ+FroMlIVzJKrl3KN5luPM8FAj2CGwjoX2j6V5RlAl99lkKce8KSp76HVM
           ItQVFGP94wfzMxeV1MFN5lYYJvOyApDgFLh9B35/Y2n2XcBu8KytvUR0zPojH3Q8YsGXR
           y8MeCclG92oucn4huTRMTFdfuUuuAVRXapUMOP7M8HP8tqeYQkQjK1ipH2RyFE=
-      pco_prod3_key: !encrypted/pkcs1-oaep
-        - E5OhAWzDwWDYcCCgn5gVu4ZUfkej48S7hvIZIRmZj2VHNjXQaJ72ZRnl2aqdbRUTnLcfj
-          vhbAuWUR+WSaPkN16rRtvUahm7LrvpoBy2gx+3Pv5dJBuKw++4/hHpQJpPm7eFdpgSdY+
-          LR9ZDz38NrpCKWcZO1pmTOUjSnT2HUSEN4jgdZrU0Mmxo4A1XcNF4JRwAc3F6uGyRAN0H
-          mTDuNV+fv4sWaMRgRpIpzSWf0YWauZINojVQoyRhsum++I4eiUYP1BNxEr5y4V2bgjZOT
-          gzWJxVeo+9h/Cl2mQ5Pl/+0uzWNOWwG+bvH9xjpXl2+7vBSxTd04NZyOx+9GDPpYcAoZt
-          1/QxIWqvTstFHQWu5pN1CRdi5NyI5R+ifp3dGxUKU/HAtIXt/nDtUuw2uTAI1/Axq6eA9
-          3BFl9EWb62WfSVmu8qfhhTK778Iq4PYfjM4RHTrMJNlRUVWiBPZSOJ7iXaVHm5W5NPOGt
-          fbpDiiqWhglo0ki7oxmzexhKtJjwwmVpNXT/auxZnTKgw3IVCFawdE5EYDyhXff1CxkJQ
-          X1H0AovZoZqC7M2/SDftBH2phaDz2rTvzrFbH4ScQdGZ+qGNA86yP6IWa9XGk2zNOZVlF
-          w01twRJeCgmTehPtP5KwY1Y0e0cm18h03dXU2Z4FL9NKRb5j4TfFx0gTVJurjY=
       pco_prod3_ac_id: !encrypted/pkcs1-oaep
         - Sxo7/4v13s46G/j45jkBrvI7puOww3VeKwc2wIaTGw7QIABH/wpcECcUqklXePd6iOWIU
           apTY8h8uRd+3HyKn5z9b60L0QuSrkE7pawwN544CpFNMq2uImvfv4b/VeLk1VrJn/zO5j
@@ -255,17 +189,6 @@
           riPv16TIXHG/VYteDxT4f/onB6xBBiZ7Bm7drM3nAsM+ZM3WwxUnu4luWZsZYAADV+S31
           ODoxs6vEGmQgoOCCej1a71jkLMx+xdOeRN606H6Jrqfm2BfsYa1ZxUhX1Dk1dgpcVXZOK
           gJLXG3zz2PJNa/Zl0/3aqrWJS6+A9lD5XuDHDdPxKfyhwo+R2+zhzScd/bbn4E=
-      pco_prod4_key: !encrypted/pkcs1-oaep
-        - mSkFp4c/Cro+fNqBJNaMMt/VyTXmy9HNnNuKLowZDgxt0H40g+o/soLLQdRwwNbh1ktI6
-          bbOhbruQ2UvMi+pnJIR4TwSvkxHqO3n9thGAYVIGAYuuidZpdW/VVR8ISxrnmOiPGFCXo
-          G0YgdgLRb7yNh2Wn840UHKceYwVdaovPifF8KOaCXjt5t8b9+64DBJqYjAQPHDkat0Pwh
-          GQPnN+FUpoBTxUh4KIjD8yl+qXJVMaZ6/00A1b9GxK1gq2vbLqpK1EQvTZlLduI5ckrXe
-          lCUrz/OUXieGw/tKDyCpbNVmkgZAvdxCNea04vzKogJjIak+jFXN0vK7vT05hfGJRPbaa
-          tuGW32ChiTj6zoAmJoNFJHh0DmV5/C1gFMCFTTXNfhoSpigTb841zHHR/DE2zluzVbope
-          WqsnrG6iohKZ2ZhG+p96wlOG6WfS1zWkl0sNafrw35Nqz6qd7dWnmOJffOtW6WuNyuXDH
-          VfHQSOSy9hDC+tbt87xwpWNbWIMV9wVFPNLMowb9oAmqbjEEV2qMR9t+XJFHGQ3ta+nFM
-          IPiM1gXvIZEXHRW23JBckC4dgxWZ43F2u7TM6emB2gW1DKgnWOn/tRlIcNKZxeZ6daJ9Y
-          s8QePn/Z5DS6DaADyyLGaNbIAUXTvEgCEZvq6xAknTF0PT0zqw33PF3mwNoclM=
       pco_prod4_ac_id: !encrypted/pkcs1-oaep
         - L4tJH+zPSVZweHeg7FjSVgeDZdumMqhyEU9Amf6lUKqrHGz7llHgDp0InKyjrFe/CwWkG
           Y3hySGiEvsrdqywYWRq3y1gfxCvdJ7RMIO7j0xH2oJtCa+v1MpJYLG7FwC34YNt4aphgg
@@ -288,17 +211,6 @@
           +sTx+tASbZ9UobexgC+3hyMa1CanFzPPjgMm3UYyrMmnvi96zImau6Q/CpJhQg3tZ8vLz
           4BnqOQklRAJxZA5btw8SFAb7GB2TCeEs/+dt/XqLrY2XkeaR9lGBl3Bftvkr9vFVfsVmx
           7IMobRXhnMOdUZQo7JBc5BV2CB0ZhBn0phUCHQtD4BGQZb/YIl0wO1wyJdk4A0=
-      regio_a_key: !encrypted/pkcs1-oaep
-        - UEDFCkodx6dlfe9bidIhoPdXqEY4vBT9rwJLXXveBmPY9Q3cnQkQRjz4D/o7VHkyfCpkj
-          hzWgvxpsFKnVBkHgLNCbXH8YUhhDTfNGJeLvgVNMo1sk/3JdfUynvgPNAWo1IA9hxxgXN
-          J1NcpDSeASuZ1fPdxO5LzSP6SFDOS3dkX3C71RQ4A8oBSLJRxH0sQYgn/dMz4I0IF98iG
-          Vqr4KDd0yoBcAPOQyefc0sgmMphPM/pNfCFrAGLc5SpEsulvFdlKZ6xpAvh5b86wYqCmH
-          S1ou+KFG0lVr3b2g6sdiOV7BjhAyc7EQyfcvTcMTgP6mPNoraGJeoybtvshKg6OfxxskK
-          dJUJ0fXkdmXzKbTsfjsBi4wDnbQ1FmFR6WbvJfNQTmOu6x5Q2qqcxU/fTfQCyQgBMZv6S
-          fglIrVDxCeQMxJeNrVedV8/nC8pjbA4+CJQ9XVLW85yE9/7xUBkk7ABRmXI88NH9Y43Q+
-          nN49Lj5HApjqqEga2AoMRKYmm/t/bcDIurACBi2SwkTcAwP7oYam7+W/LA5rxYhMHDph5
-          uPd2Sqo777+OS3tWmYruGpG3tu4y0XHqbzmXdsRpHyQyHqKqu4YBTY6PXnFAKfQPblsBd
-          0AOUu/BCM2jTSwN4uLplWZ/+uMTKxb7styT8TQd6bK3qt1WDKqIypwQSThnWEQ=
       regio_a_ac_id: !encrypted/pkcs1-oaep
         - lBlWjvJ6RA3uniS1M4etvbdUxKB9KRNWm53gL3VlPyRkA7Ic2yFcAkZEGodHWH1iqNWfN
           p/3B2iFYwuZRktllbc/Ro80pkg52vHOkNkBdXLQd7ZFKG8zNJOxRt2nhDQxQPS7PzcUgo
@@ -321,17 +233,6 @@
           VCsXjf0qBBMrzz6HP9z95Bk44fiJ3L/LkA3Iij961dYrQXbZKDrKOiX/QPwrcSrVmjmew
           UbPexJFHgvTCqjadoLejSt9cUd9lVzhuzLJ8CS+CcCMbZOno6qathrd2B88riQaPNIGNu
           gfkNT9R63ZzKB1qIA2n5RZi7SH9DPIUd0AwLMn2bhp3uok5pNAPP/4/1RkQiCA=
-      syseleven_dus2_key: !encrypted/pkcs1-oaep
-        - OR+8FJcBf56cPm0oTCs8T4NoynwyqA6cjHYlWzD35YX9zJaZoZgiI5MVn4BjtvaXpQfR6
-          M0ZOZYHHZqbDLGAXtZcm26uEVcu1GUfbcCrgnfg71Mjk3vucLeSxfxuAzLJ7ZxLjAjIjD
-          pzytvEngjZga4thyhMujQoBWvmlrBjUtwnDNyM8bn5LIA8wnCu5nMb6KrqVR1xgQPUS90
-          xcvRDVN+e8cYvYRcn4u5LT75JUGbCjXz1l0N15hH5+0uqiZEOHllj+Ed+JSHvTR0twnwA
-          ECtwe0I6MqakqOi4SLuDfg5YOnhzrXzPq6MaKPxrbsai7nqI5odjVwwNO7kf8GsHObKIR
-          t6lJNqklJP8SSkp3gl6zZBXD7nVOtfCd4BGPV+k5h/TCOW7Jnkrp8eFhgs1RCptEUYe9p
-          3Cvphzbvi2uUFQOmR5tcDLbkSZa2fG79AmxRZKDUVHrf7gEaYqNpEYJjG5KzNdFVg8XVM
-          EyXFhoEdGj7k+VGrAwjt1ohJUWKSwnM67eN0A6na4pMGmUuO+w8/mxPJGdKGEqEIFhKuq
-          NJdwuSJ7ANdnHFV/1PkHFkWRG0RLsjvWXBjlA5BH8P5nVBGPuVdXt0rnxSlqQxSkc6Z0n
-          kk3eqIOeLVlVrERPSIoJmrfd7dZhRD3JXsVCHUr5nlU4IJ3MOy8LJeNhZ7dm8U=
       syseleven_dus2_ac_id: !encrypted/pkcs1-oaep
         - SjwtIvJO7DkLJDmS+T/Z5utFBa22hmPRBd8mzonJHGgURB2W7fmXFreD9NPrLfbt7ujKi
           KNqJm8k1Vr1F3Mu+Osr0BWSnq5makwVt2ikBY4qPbL8iyVXsByaT/HNPLCOokqy+REpfu
@@ -354,17 +255,6 @@
           8tYIxiQD85Rnv431bk8yq/Cc8s0Q77SWZNln1NieDnOyvXALwvE1owUrC7NnMuIcGaySk
           so/qIZP35Hs2rGR/LWVLIpQco/RzI6KaSN5Idk6906Q2UCgNErvmKQAnGaA+CUCkRqodX
           nw8TQSmx0VChVFhMnCevWTl2vHT3KrAb7NtOV53WG0RFvL220WkEeT3Fllb1LE=
-      syseleven_ham1_key: !encrypted/pkcs1-oaep
-        - tDdSLg+kLf8Mx9efsTnIZO/X1WflI/8uXTjbwDg0Lxyd22hV1UImGQPhLpJDGO9QoCpLX
-          Q94UG3O7JrO8DWJKS/diXod4L6BMwX9MnSsZ/5OHJDCycznkj/KbO9Ko0ry9ICLiAUklb
-          voXqZ6tHX4jpBUJZI7VghXu1vbbDVFb4vugbMZdhzXSUnEZck/7+8CdVLVIDAMCNGZmy9
-          7CgIeKMrm/inK3NTPrgmgG5v+ZtIwzrU3MAK6OSHCb02bI1ThRmsaAPeGb77x4GBfKygC
-          zeItgCrVJBnBP9GrKgjQAlCr/+gezEppN2WTxWJ95tRkSALg+BnCqI0RS3oEUnBceGHoF
-          N2lgo5/dqFRNX180w6w/In3Za1ChLsBImULJm9FSp12FPIGlVTMVuSVpeyi/rvdWmxNiN
-          3Y1tfpkgPsckmOmN8vtud6kogJj6OKeCfdS534QGg54z8qYoh2PO3IEJ0uibramexo+NI
-          bimkBmbshkY2FNLU5JEFI5ba+YWCF/VqGdPvarR4dmBBHe8enPCKi4DUFnJt9XLeNakUS
-          D4caxPISJ1MAmYBAcOJZwSH1ODjmrPD/nTtLpFILdUHiYb+D3GVpTGQZTeFKfpENCkhxD
-          LAR4HytFA4Uf6c2AVoF1sw8TQa94H4k7WBAS3BZYVPJJSyuJDXSf+6874HGg4c=
       syseleven_ham1_ac_id: !encrypted/pkcs1-oaep
         - SjwtIvJO7DkLJDmS+T/Z5utFBa22hmPRBd8mzonJHGgURB2W7fmXFreD9NPrLfbt7ujKi
           KNqJm8k1Vr1F3Mu+Osr0BWSnq5makwVt2ikBY4qPbL8iyVXsByaT/HNPLCOokqy+REpfu
@@ -387,17 +277,6 @@
           8tYIxiQD85Rnv431bk8yq/Cc8s0Q77SWZNln1NieDnOyvXALwvE1owUrC7NnMuIcGaySk
           so/qIZP35Hs2rGR/LWVLIpQco/RzI6KaSN5Idk6906Q2UCgNErvmKQAnGaA+CUCkRqodX
           nw8TQSmx0VChVFhMnCevWTl2vHT3KrAb7NtOV53WG0RFvL220WkEeT3Fllb1LE=
-      wavestack_key: !encrypted/pkcs1-oaep
-        - RcNRv65kND4PZ7yGsQMLjQ8TuWhQYvQ24hmiSf6/OfEgMK036CUn8pgFmznVGbAMc0SVZ
-          1pznRqADDCqdRx18aSlpcN0Sk7U8dzVtxZoKs3PxKMFm0OmZZBJYGs+nBdZOf8pYGbLOU
-          IbxsnxbY/DQ0pemK46XmzGAARtiHk2VEOEB1Ai0KgZdk0+F4ifJv0w97PkDsMk436xyqY
-          Z44gpuFJvtkV1J+zYpmQI9ShAOVfTR8r/ZXoyrklPZrbr/ZUkUF0b5Soi7HPBKSDcDdov
-          7l8BXh6HUJwGzhieu9Ln/G/PCDysDv8x+US3i70jKRoaJhtoM7HDdWDFzprRF9fIJwAc5
-          8gsSAqYYJ4eBATmaGZDTBpIiHahkMlaGHDp5gOG1aHoDR5NpQ/OYYIvQNiuxbViMdq7r/
-          TslHC030nv4nLkpy9T5c2GMLTaDKsrGlwYJahMHRnXVYzX07C6h7Nhm7mGClzHLP2OtAl
-          p32VmEYjs/Kl+7vBPW1k4Av0Wu9fbul40SmgdK6B/Cdi6sQ/NOfMNlgPAj1Kp6WVRgpRw
-          84fxvXN134x+e1JiDqkl6f0aesS6jSwxy6oh89t2cDMDfdNG4vS33APKiqm1BemAGGRJa
-          MDNlNXXoUaJ1C6inLCGOtTbQGylkDiCGO1laCaZbELDMaoEEBG+3vIAKIJSiDE=
       wavestack_ac_id: !encrypted/pkcs1-oaep
         - NgtWt9AeOFCvfDaDtYdWAFO1oh+LVLMNi2gyK2N0IHkf5SK68DRkR8asKm10iOIaXVkN4
           riQQqirjYHzIzWS2s/dKoLIH5DTpRHZUl4n8i7sdN5lhdoxjga5+Ep+FWTG8oSWN6ZJFP

Tests/cleanup.py

@@ -202,6 +202,7 @@ def main(argv):
         level=logging.INFO,
         datefmt="%Y-%m-%d %H:%M:%S",
     )
+    openstack.enable_logging(debug=False)
 
     prefix = os.environ.get("PREFIX", None)
     cloud = os.environ.get("OS_CLOUD")

Tests/config.toml

@@ -25,6 +25,7 @@ subjects = [
     "pco-prod4",
     "poc-kdo",
     "poc-wgcloud",
+    "regio-a",
     "syseleven-dus2",
     "syseleven-ham1",
     "wavestack",

Tests/scs-test-runner.py

@@ -26,6 +26,7 @@ class Config:
     def __init__(self):
         self.cwd = os.path.abspath(os.path.dirname(sys.argv[0]) or os.getcwd())
         self.scs_compliance_check = os.path.join(self.cwd, 'scs-compliance-check.py')
+        self.cleanup_py = os.path.join(self.cwd, 'cleanup.py')
         self.ssh_keygen = shutil.which('ssh-keygen')
         self.curl = shutil.which('curl')
         self.secrets = {}
@@ -70,6 +71,16 @@ def build_check_command(self, scope, subject, output):
             cmd.extend(['-a', f'{key}={value}'])
         return cmd
 
+    def build_cleanup_command(self, subject):
+        # TODO figure out when to supply --debug here (but keep separated from our --debug)
+        return [
+            sys.executable, self.cleanup_py,
+            '-c', self.get_subject_mapping(subject)['os_cloud'],
+            '--prefix', '_scs-',
+            '--ipaddr', '10.1.0.',
+            '--debug',
+        ]
+
     def build_sign_command(self, target_path):
         return [
             self.ssh_keygen,
@@ -176,6 +187,33 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml):
     return 0
 
 
+@cli.command()
+@click.option('--subject', 'subjects', type=str)
+@click.option('--preset', 'preset', type=str)
+@click.option('--num-workers', 'num_workers', type=int, default=5)
+@click.pass_obj
+def cleanup(cfg, subjects, preset, num_workers):
+    """
+    clean up any lingering resources
+    """
+    if not subjects and not preset:
+        preset = 'default'
+    if preset:
+        preset_dict = cfg.presets.get(preset)
+        if preset_dict is None:
+            raise KeyError('preset not found')
+        subjects = preset_dict['subjects']
+        num_workers = preset_dict.get('workers', num_workers)
+    else:
+        subjects = [subject.strip() for subject in subjects.split(',')] if subjects else []
+    if not subjects:
+        raise click.UsageError('subject(s) must be non-empty')
+    logger.debug(f'cleaning up for subject(s) {", ".join(subjects)}, num_workers: {num_workers}')
+    commands = [cfg.build_cleanup_command(subject) for subject in subjects]
+    _run_commands(commands, num_workers=num_workers)
+    return 0
+
+
 if __name__ == '__main__':
     logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
     cli(obj=Config())