Containerisation of Continuous Integration (#27)

* Changes to scripts for github actions

* rootfs ci without pushing image to registry

* fixed job name error

* fixed dockerfile path error

* chnages to dockerfile context

* dockerfile location change

* solving issue with github workspace

* trying to solving issue with github workspace

* trying to solving issue with dockerfile again

* removing lines from dockerfile

* specifying branches

* adding .dockerignore file

* changing context path

* changing build push syntax

* move dockerfile out of rootfs

* add dockerfiles

* remove copying step

* artifact location

* artifact location fixed

* artifact location fixed this time

* artifact location attempt

* docker load loacation fixed

* artifact path change

* artifact path change to inside github workspace

* artifact path change to inside github workspace

* Remove interactive container option

* chnages to docker run command

* changes to docker run command to make it work

* removing modes for the volumes

* add platform to make command

* adding config file to remote

* changing zip artifact location

* changing build location

* cleaning up and adding more comments

* Adding more repos to github workspace

* Add release of container to ghcr

* chnage repo name to have no capitals

* changing image name

* fixing name of tar file in release job

* Adding tags for ghcr

* changing tags for image

* making sure docs pull image correctly

* fixing ghcr login for docs ci

* using sudo to move html directory

* Cleaning up the docs yml

* moving html directory

* changing username

* replacing link to i2c tar file

* changing rclone command

* path changes for scripts

* changing rclone installation

* rclone patch

* rclone patch script change

* install fuse

* remove rclone from container

* changing name of platform

* Replace paths in CONFIG.example

* Replace link to dropbear tar file

* Simpify Docs publishing

* Implement uploading container with Tagged version

* Updated conf.py for new version of Sphinx

* Correct image name in metadata

* Checkout sources for rebuild and release of container

* Remove outputs argument from releasing image

* Correct container name in Metadata

* Flag pre-release tags

Co-authored-by: Tizayi Zirereza <[email protected]>
Co-authored-by: root <[email protected]>

.dockerignore

@@ -0,0 +1,3 @@
+#Ignore the git and cache folders
+.git
+.cache

.github/scripts/GNU-toolchain.sh

@@ -1,19 +1,17 @@
 #!/bin/bash
-# Installs and extracts a GNU toolchain based on the systems architecture
 
-PLATFORM=$1
+# Put in own dir
+# Installs and extracts a GNU toolchains for both systems architectures
+# ARM GNU Toolchains
+TOOLCHAIN_ZYNQ=https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/gcc-arm-10.2-2020.11-x86_64-arm-none-linux-gnueabihf.tar.xz
+TOOLCHAIN_ZYNQMP=https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz
 
-# ARM GNU Toolchain
-# Determine which toolchain to use
-if [ "$PLATFORM" == "zynq" ]; then
-    TOOLCHAIN=https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/gcc-arm-10.2-2020.11-x86_64-arm-none-linux-gnueabihf.tar.xz
-elif [ "$PLATFORM" == "zynqmp" ]; then
-    TOOLCHAIN=https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz
-else >&2 echo "Error: Argument $PLATFORM must be either zynq or zynqmp"
-    exit 1
-fi
+# Download and extract both toolchains
+curl -o toolchain-zynq.tar.xz -L $TOOLCHAIN_ZYNQ
+curl -o toolchain-zynqmp.tar.xz -L $TOOLCHAIN_ZYNQMP
 
-# Download and extract toolchain
-curl -o toolchain.tar.xz -L $TOOLCHAIN
-tar -xf toolchain.tar.xz
-rm toolchain.tar.xz
+tar -xf toolchain-zynq.tar.xz
+tar -xf toolchain-zynqmp.tar.xz
+
+rm toolchain-zynq.tar.xz
+rm toolchain-zynqmp.tar.xz

.github/scripts/config-file-rootfs.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
 # Generates and populates CONFIG file for rootfs repo.
 
-cat >> rootfs/CONFIG.local << 'EOL'
+cat >> /rootfs/CONFIG.local << 'EOL'
 TARGET = minimal
 
 # This is the location where source and build files will be placed.
-ROOTFS_ROOT = $(GITHUB_WORKSPACE)/build
+ROOTFS_ROOT = /build
 
 # This is where all of the source tar files will be found.
-TAR_DIRS = $(GITHUB_WORKSPACE)/tar-files
+TAR_DIRS = /tar-files
 EOL

.github/scripts/rclone-patch.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Installation of go & temporary rclone patch applied until symlinks issue using rclone mount is resolved (see below):
+# https://github.com/rclone/rclone/issues/2975
+
+#Install go
+curl -OL https://golang.org/dl/go1.17.linux-amd64.tar.gz
+rm -rf /usr/local/go && tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz
+export PATH=$PATH:/usr/local/go/bin
+go version
+
+#Install rclone patch
+git clone https://github.com/PandABlocks/rclone.git
+cd rclone
+git checkout traack_symlinks
+make
+./rclone version
+cd home/runner/go/bin/
+cp rclone /usr/bin/rclone
+chown root:root /usr/bin/rclone
+chmod 755 /usr/bin/rclone
+mkdir -p /usr/local/share/man/man1
+cp rclone /usr/local/share/man/man1/
+mandb
+rclone version
+

.github/scripts/tar-files.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 # Installs all sources relevant to the rootfs build as tar files
 
-# Create the tar-files directory
-mkdir tar-files
-cd tar-files
+# Enter the tar-files directory
+mkdir /tar-files 
+cd /tar-files
 
 # If no argument is given
 if [[ -z $1 ]]; then
-    # Install tar file dependencies
+    # Install tar file rootfs dependencies
     curl -OL https://github.com/libffi/libffi/releases/download/v3.3-rc2/libffi-3.3-rc2.tar.gz \
         -OL https://ftp.gnu.org/gnu/nano/nano-2.4.1.tar.gz \
         -o cothread-2-18.tar.gz -L https://github.com/dls-controls/cothread/archive/refs/tags/2-18.tar.gz \
@@ -20,7 +20,7 @@ if [[ -z $1 ]]; then
         -OL https://qa.debian.org/watch/sf.php/e2fsprogs/e2fsprogs-1.46.2.tar.gz \
         -OL https://github.com/numpy/numpy/releases/download/v1.17.5/numpy-1.17.5.tar.gz \
         -OL http://www.eecis.udel.edu/~ntp/ntp_spool/ntp4/ntp-4.2/ntp-4.2.8p15.tar.gz \
-        -OL https://matt.ucc.asn.au/dropbear/releases/dropbear-2015.67.tar.bz2 \
+        -OL https://mirror.dropbear.nl/mirror/releases/dropbear-2015.67.tar.bz2 \
         -OL https://ftp.gnu.org/gnu/readline/readline-6.3.tar.gz \
         -OL https://busybox.net/downloads/busybox-1.33.1.tar.bz2 \
         -o tornado-6.0.3.tar.gz -L https://github.com/tornadoweb/tornado/archive/refs/tags/v6.0.3.tar.gz \
@@ -29,7 +29,7 @@ if [[ -z $1 ]]; then
         -OL https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-3.0.2.tar.gz \
         -OL https://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.gz \
         -OL https://files.pythonhosted.org/packages/11/0a/7f13ef5cd932a107cd4c0f3ebc9d831d9b78e1a0e8c98a098ca17b1d7d97/setuptools-41.6.0.zip \
-        -OL http://jdelvare.nerim.net/mirror/i2c-tools/i2c-tools-3.1.1.tar.bz2 \
+        -OL http://oe-lite.org/mirror/i2c-tools/i2c-tools-3.1.1.tar.bz2 \
         -OL https://ftp.gnu.org/gnu/screen/screen-4.2.1.tar.gz \
         -OL https://ftpmirror.gnu.org/libtool/libtool-2.4.6.tar.gz \
         -OL https://ftp.gnu.org/gnu/ncurses/ncurses-6.1.tar.gz \
@@ -40,3 +40,22 @@ fi
 # Install linux tar file dependencies
 curl -o u-boot-xlnx-xilinx-v2020.2.2-k26.tar.gz -L https://github.com/Xilinx/u-boot-xlnx/archive/refs/tags/xilinx-v2020.2.2-k26.tar.gz \
     -o linux-xlnx-xilinx-v2020.2.2-k26.tar.gz -L https://github.com/Xilinx/linux-xlnx/archive/refs/tags/xilinx-v2020.2.2-k26.tar.gz
+
+# Install tar file dependencies for PandAblocks-FPGA 
+curl -o device-tree-xlnx-xilinx-v2020.2.tar.gz -L https://github.com/Xilinx/device-tree-xlnx/archive/refs/tags/xilinx-v2020.2.tar.gz \
+    -o u-boot-xlnx-xilinx-v2020.2.tar.gz -L https://github.com/Xilinx/u-boot-xlnx/archive/refs/tags/xilinx-v2020.2.tar.gz \
+    -o arm-trusted-firmware-xilinx-v2020.2.tar.gz -L https://github.com/Xilinx/arm-trusted-firmware/archive/refs/tags/xilinx-v2020.2.tar.gz \
+    -o dtc-1.6.1.tar.gz -L https://git.kernel.org/pub/scm/utils/dtc/dtc.git/snapshot/dtc-1.6.1.tar.gz
+
+# Old url to i2c-tools tar file. No longer working
+# -OL http://jdelvare.nerim.net/mirror/i2c-tools/i2c-tools-3.1.1.tar.bz2 \
+
+# 2021.1 tar files
+curl -o u-boot-xlnx-xilinx-v2021.1.tar.gz -L https://github.com/Xilinx/u-boot-xlnx/archive/refs/tags/xilinx-v2021.1.tar.gz \
+    -o linux-xlnx-xilinx-v2021.1.tar.gz -L https://github.com/Xilinx/linux-xlnx/archive/refs/tags/xilinx-v2021.1.tar.gz
+
+
+ curl -o device-tree-xlnx-xilinx_v2021.1.tar.gz -L https://github.com/Xilinx/device-tree-xlnx/archive/refs/tags/xilinx_v2021.1.tar.gz \
+    -o u-boot-xlnx-xilinx-v2021.1.tar.gz -L https://github.com/Xilinx/u-boot-xlnx/archive/refs/tags/xilinx-v2021.1.tar.gz \
+    -o arm-trusted-firmware-xlnx_rebase_v2.4_2021.1.tar.gz -L https://github.com/Xilinx/arm-trusted-firmware/archive/refs/tags/xlnx_rebase_v2.4_2021.1.tar.gz \
+#    -o dtc-1.6.1.tar.gz -L https://git.kernel.org/pub/scm/utils/dtc/dtc.git/snapshot/dtc-1.6.1.tar.gz