Merge pull request #1 from highwave-project/basilisk-install

Basilisk install workflow build successfully but fails on testing
highwave-project · Feb 15, 2024 · 8c5cafc · 8c5cafc
2 parents 3b05923 + eb95f5c
commit 8c5cafc
Show file tree

Hide file tree

Showing 2 changed files with 79 additions and 48 deletions.
diff --git a/README.md b/README.md
@@ -190,11 +190,11 @@ Submit a job array with index values between 1 and 7 with a step size of 2 (i.e.
 
 Each job has the following environment variables set:
 
-1. **`SLURM_ARRAY_JOB_ID`**: The first job ID
-1. **`SLURM_ARRAY_TASK_ID`**: The job array index value
-1. **`SLURM_ARRAY_TASK_COUNT`**: The number of tasks in the job array
-1. **`SLURM_ARRAY_TASK_MAX`**: The highest job array index value
-1. **`SLURM_ARRAY_TASK_MIN`**: The lowest job array index value
+1. `SLURM_ARRAY_JOB_ID`: The first job ID
+1. `SLURM_ARRAY_TASK_ID`: The job array index value
+1. `SLURM_ARRAY_TASK_COUNT`: The number of tasks in the job array
+1. `SLURM_ARRAY_TASK_MAX`: The highest job array index value
+1. `SLURM_ARRAY_TASK_MIN`: The lowest job array index value
 
 For example the following job submission will generate 3 jobs:
 
@@ -362,6 +362,9 @@ To ensure issue free compilation of your basilisk code it is recommended to use
     - `export LOCAL_INSTALL=yes`: you have sudo access
     - `export BUILD_GRAPHICS=yes`: you require ffmpeg compiled, or OSMesa or GLU (This may take a long time...)
     - You likely only require one of the above, as if you have sudo access you can simply install the graphics binaries
+    - `export INSTAL_PREFIX=/basilisk/install/dir`: This is the directory where basilisk will be installed. If not set the default is your home directory, ie `$HOME`.
+    - `export DEPS_PREFIX=/dependencies/install/dir/`: This is the directory where the dependencies (ffmpeg/OSMesa/GLU) will be installed. If not set the default is `$HOME/local`
+   - `export NO_MOD_PATH=yes`: If this variable is **NOT** specified, at the end the scripts adds basilisk (and dependencies) to PATH. Additionally, the `lib` and `include` directories of the dependencies are added to the c include and linker paths. This is recommended so that you don't need to pass the flags `-I$HOME/local/include` and `-L$HOME/local/lib` every time you compile a basilisk file. (Only add them if those directories are not already in the respective paths)
 
 ```bash
 cd ~

diff --git a/install_basilisk.sh b/install_basilisk.sh
@@ -1,15 +1,28 @@
+#!/bin/bash -l
+
 JOBS=$(nproc)
+# Where the basilisk will be installed
+INSTALL_PREFIX=${INSTALL_PREFIX:-$HOME}
+# Where the dependecies (osmesa, glu, ffmpeg) will be installed
+DEPS_PREFIX=${DEPS_PREFIX:-"$HOME/local"}
 
 if [[ ! -z $TESTING ]]; then
     source $HOME/.bashrc
     echo "Testing only"
     cd $BASILISK/test
-    make -j "$JOBS" >/dev/null || { echo 'testing failed' && exit 1; }
+    make -j "$JOBS" || { echo 'testing failed' && exit 1; }
     exit 0
 fi
 
 # clean up old folders
-rm -rf basilisk* ffmpeg* bin* "local*" mesa* glu* 
+rm -rf $INSTALL_PREFIX/basilisk* $DEPS_PREFIX/ffmpeg* $DEPS_PREFIX/mesa* $DEPS_PREFIX/glu*
+if [[ ! -d $DEPS_PREFIX ]]; then
+    mkdir $DEPS_PREFIX
+fi
+if [[ ! -d $INSTALL_PREFIX ]]; then
+    mkdir $INSTALL_PREFIX
+fi
+
 # Install packages
 if [[ ! -z $LOCAL_INSTALL ]]; then
     echo "Local installation with access to sudo for installing"
@@ -24,7 +37,8 @@ if [[ ! -z $LOCAL_INSTALL ]]; then
 else
     echo "Installation with no access to sudo"
     wget http://basilisk.fr/basilisk/basilisk.tar.gz
-    tar xzf basilisk.tar.gz >/dev/null
+    tar xzf basilisk.tar.gz -C $INSTALL_PREFIX >/dev/null
+    rm basilisk.tar.gz
 fi
 
 # if on a cluster with module, check if the following can be loaded
@@ -33,7 +47,7 @@ if which module; then
 fi
 
 echo "Building Basilisk..."
-cd basilisk/src
+cd $INSTALL_PREFIX/basilisk/src
 ln -s config.gcc config
 make -k -j "$JOBS" >/dev/null
 make  # incase of any failures from previous command
@@ -44,106 +58,120 @@ else
     shellrc="$HOME/.bashrc"
 fi
 
-if ! grep -q 'BASILISK' $shellrc; then
-    echo "Export environment variables to shell config"
-    echo -e '\n#Basilisk Env' >> "$shellrc"
-    echo "export BASILISK=$PWD" >> "$shellrc"
-    echo 'export PATH=$PATH:$BASILISK' >> "$shellrc"  # single quotes to not expand $PATH
-fi
-
 if [[ ! -z $BUILD_GRAPHICS ]]; then
     echo "Graphics building enabled..."
 
     if ! which ffmpeg; then
-        mkdir $HOME/ffmpeg_sources
+        mkdir $DEPS_PREFIX/ffmpeg_sources
 
         if ! which nasm; then # Install asm compiler
             echo "---------------- Asm"
-            cd $HOME/ffmpeg_sources
+            cd $DEPS_PREFIX/ffmpeg_sources
             wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 >/dev/null
             tar xjvf nasm-2.15.05.tar.bz2 >/dev/null
             cd nasm-2.15.05 
             ./autogen.sh >/dev/null
-            PATH="$HOME/bin:$PATH" ./configure --prefix="$HOME/ffmpeg_build" --bindir="$HOME/bin" >/dev/null
+            PATH="$DEPS_PREFIX/bin:$PATH" ./configure --prefix="$DEPS_PREFIX/ffmpeg_build" --bindir="$DEPS_PREFIX/bin" >/dev/null
             make -j "$JOBS" >/dev/null || { echo 'nasm build failed' && exit 1; }
             make install >/dev/null || { echo 'nasm install failed' && exit 1; }
         fi
 
         if ! which x264; then # Install support for x264 video encoding
             echo "---------------- x264"
-            cd $HOME/ffmpeg_sources && \
+            cd $DEPS_PREFIX/ffmpeg_sources && \
             git clone --depth 1 https://code.videolan.org/videolan/x264.git >/dev/null
             cd x264 && \
-            PATH="$HOME/bin:$PATH" PKG_CONFIG_PATH="$HOME/ffmpeg_build/lib/pkgconfig" ./configure --prefix="$HOME/ffmpeg_build" --bindir="$HOME/bin" --enable-static --enable-pic >/dev/null
-            PATH="$HOME/bin:$PATH" make -j "$JOBS" >/dev/null || { echo 'x264 build failed' && exit 1; }
+            PATH="$DEPS_PREFIX/bin:$PATH" PKG_CONFIG_PATH="$DEPS_PREFIX/ffmpeg_build/lib/pkgconfig" ./configure --prefix="$DEPS_PREFIX/ffmpeg_build" --bindir="$DEPS_PREFIX/bin" --enable-static --enable-pic >/dev/null
+            PATH="$DEPS_PREFIX/bin:$PATH" make -j "$JOBS" >/dev/null || { echo 'x264 build failed' && exit 1; }
             make install >/dev/null || { echo 'x264 install failed' && exit 1; }
         fi
 
         if ! which x265; then # Install support for x265 video encoding
             echo "---------------- x265"
-            cd $HOME/ffmpeg_sources && \
+            cd $DEPS_PREFIX/ffmpeg_sources && \
             wget -O x265.tar.bz2 https://bitbucket.org/multicoreware/x265_git/get/master.tar.bz2 >/dev/null
             tar xjvf x265.tar.bz2 >/dev/null
             cd multicoreware*/build/linux && \
-            PATH="$HOME/bin:$PATH" cmake -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX="$HOME/ffmpeg_build" -DENABLE_SHARED=off ../../source >/dev/null
-            PATH="$HOME/bin:$PATH" make -j "$JOBS" >/dev/null || { echo 'x265 build failed' && exit 1; }
+            PATH="$DEPS_PREFIX/bin:$PATH" cmake -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX="$DEPS_PREFIX/ffmpeg_build" -DENABLE_SHARED=off ../../source >/dev/null
+            PATH="$DEPS_PREFIX/bin:$PATH" make -j "$JOBS" >/dev/null || { echo 'x265 build failed' && exit 1; }
             make install >/dev/null || { echo 'x265 install failed' && exit 1; }
         fi
 
         echo "---------------- FFMPEG"
-        cd $HOME/ffmpeg_sources && \
+        cd $DEPS_PREFIX/ffmpeg_sources && \
         wget -O ffmpeg-snapshot.tar.bz2 https://ffmpeg.org/releases/ffmpeg-snapshot.tar.bz2 >/dev/null
         tar xjvf ffmpeg-snapshot.tar.bz2 >/dev/null
         cd ffmpeg && \
-        PATH="$HOME/bin:$PATH" PKG_CONFIG_PATH="$HOME/ffmpeg_build/lib/pkgconfig" ./configure \
-        --prefix="$HOME/ffmpeg_build" \
+        PATH="$DEPS_PREFIX/bin:$PATH" PKG_CONFIG_PATH="$DEPS_PREFIX/ffmpeg_build/lib/pkgconfig" ./configure \
+        --prefix="$DEPS_PREFIX/ffmpeg_build" \
         --pkg-config-flags="--static" \
-        --extra-cflags="-I$HOME/ffmpeg_build/include" \
-        --extra-ldflags="-L$HOME/ffmpeg_build/lib" \
+        --extra-cflags="-I$DEPS_PREFIX/ffmpeg_build/include" \
+        --extra-ldflags="-L$DEPS_PREFIX/ffmpeg_build/lib" \
         --extra-libs="-lpthread -lm" \
         --ld="g++" \
-        --bindir="$HOME/bin" \
+        --bindir="$DEPS_PREFIX/bin" \
         --enable-gpl \
         --enable-libx264 \
         --enable-libx265 >/dev/null
-        PATH="$HOME/bin:$PATH" make -j "$JOBS" >/dev/null || { echo 'ffmpeg build failed' && exit 1; }
+        PATH="$DEPS_PREFIX/bin:$PATH" make -j "$JOBS" >/dev/null || { echo 'ffmpeg build failed' && exit 1; }
         make install >/dev/null || { echo 'ffmpeg install failed' && exit 1; }
     fi
 
-    cd $HOME
+    cd $DEPS_PREFIX
+    export CFLAGS="-fcommon"    # https://gitlab.freedesktop.org/mesa/mesa/-/issues/3298
     echo "---------------- OSMESA"
     wget http://basilisk.fr/src/gl/mesa-17.2.4.tar.gz >/dev/null
     tar xzvf mesa-17.2.4.tar.gz >/dev/null
     cd mesa-17.2.4
-    ./configure --prefix=$HOME/local --enable-osmesa \
+    ./configure --prefix=$DEPS_PREFIX --enable-osmesa \
                 --with-gallium-drivers=swrast                \
-                --disable-driglx-direct --disable-dri --disable-gbm --disable-egl >/dev/null
-    make -j "$JOBS" >/dev/null || (echo 'osmesa build failed' && exit 1)
-    make install >/dev/null || (echo 'osmesa install failed' && exit 1)
+                --disable-driglx-direct --disable-dri --disable-gbm --disable-egl
+    make -j "$JOBS" || { echo 'osmesa build failed' && exit 1; }
+    make install || { echo 'osmesa install failed' && exit 1; }
 
-    cd $HOME
+    cd $DEPS_PREFIX
     echo "---------------- GLU"
     wget http://basilisk.fr/src/gl/glu-9.0.0.tar.gz >/dev/null
     tar xzvf glu-9.0.0.tar.gz >/dev/null
     cd glu-9.0.0
-    ./configure --prefix=$HOME/local >/dev/null
-    make -j "$JOBS" >/dev/null || { echo 'glu build failed' && exit 1; }
-    make install >/dev/null || { echo 'glu install failed' && exit 1; }
+    export CFLAGS="-I$DEPS_PREFIX/include"
+    export CPPFLAGS="-I$DEPS_PREFIX/include"
+    export LDFLAGS="-L$DEPS_PREFIX/lib"
+    ./configure --prefix=$DEPS_PREFIX --enable-osmesa
+    make -j "$JOBS" || { echo 'glu build failed' && exit 1; }
+    make install || { echo 'glu install failed' && exit 1; }
     cd ..
 
-    echo "export CFLAGS=-I$HOME/local/include" >> "$shellrc"
-    echo "export LDFLAGS=-L$HOME/local/lib" >> "$shellrc"
-
     echo "Cleaning up..."
-    cd $HOME && rm -rf *.tar.gz ffmpeg_sources
+    cd $DEPS_PREFIX && rm -rf *.tar.gz ffmpeg_sources mesa* glu*
 else
     echo "Graphics build disabled..."
 fi
 
-CFLAGS="-I$HOME/local/include -std=gnu99"
-LDFLAGS="-L$HOME/local/lib"
-cd $HOME/basilisk/src/ppr
+export CFLAGS="-I$DEPS_PREFIX/include -std=gnu99"
+export LDFLAGS="-L$DEPS_PREFIX/lib"
+cd $INSTALL_PREFIX/basilisk/src/ppr
 make && cd ../gl 
 make libglutils.a libfb_osmesa.a || { echo "failed building src/gl" && exit 1; }
 
+# Post installation actions
+if ! grep -q 'BASILISK' $shellrc && [[ -z $NO_MOD_PATH ]]; then
+    echo "Export environment variables to shell config"
+    echo -e '\n#Basilisk Env' >> "$shellrc"
+    echo "export BASILISK=$PWD" >> "$shellrc"
+    echo 'export PATH=$PATH:$BASILISK' >> "$shellrc"  # single quotes to not expand $PATH
+fi
+if [[ ! -z $BUILD_GRAPHICS && -z $NO_MOD_PATH ]] && [[ ":$PATH:" != *":$DEPS_PREFIX/bin:"* ]]; then
+    echo "export PATH=\$PATH:$DEPS_PREFIX/bin" >> "$shellrc"
+fi
+if [[ ! -z $BUILD_GRAPHICS && -z $NO_MOD_PATH ]] && [[ ":$LIBRARY_PATH:" != *":$DEPS_PREFIX/lib:"* ]]; then
+    echo "export LIBRARY_PATH=$DEPS_PREFIX/lib\${LIBRARY_PATH:+:\$LIBRARY_PATH}" >> "$shellrc"
+fi
+if [[ ! -z $BUILD_GRAPHICS && -z $NO_MOD_PATH ]] && [[ ":$LD_LIBRARY_PATH:" != *":$DEPS_PREFIX/lib:"* ]]; then
+    echo "export LD_LIBRARY_PATH=$DEPS_PREFIX/lib\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH}" >> "$shellrc"
+fi
+if [[ ! -z $BUILD_GRAPHICS && -z $NO_MOD_PATH ]] && [[ ":$C_INCLUDE_PATH:" != *":$DEPS_PREFIX/include:"* ]]; then
+        echo "export C_INCLUDE_PATH=$DEPS_PREFIX/lib\${C_INCLUDE_PATH:+:\$C_INCLUDE_PATH}" >> "$shellrc"
+fi
+
 echo "Installation finished..."