diff --git a/.gitignore b/.gitignore index 20c3e6e..0f04403 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ __pycache__ doc/_build *.egg-info venv +.vscode diff --git a/Dockerfile b/Dockerfile index fd820a1..7b6cb02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,273 @@ USER root RUN apt-get update && \ apt-get install -y gnome-todo apt-utils cmake make wget git +###################################################################### +# DDFacet install + +#Package dependencies +COPY apt.sources.list /etc/apt/sources.list + +#Setup environment +ENV DDFACET_TEST_DATA_DIR /test_data +ENV DDFACET_TEST_OUTPUT_DIR /test_output + +# Support large mlocks +RUN echo "* - memlock unlimited" > /etc/security/limits.conf +ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_PRIORITY critical +ENV GNUCOMPILER 7 +ENV DEB_SETUP_DEPENDENCIES \ + dpkg-dev \ + g++-$GNUCOMPILER \ + gcc-$GNUCOMPILER \ + libc-dev \ + cmake \ + gfortran-$GNUCOMPILER \ + git \ + wget \ + subversion + +ENV DEB_DEPENCENDIES \ + python3-pip \ + libfftw3-dev \ + python3-numpy \ + libfreetype6 \ + libfreetype6-dev \ + libpng-dev \ + pkg-config \ + python3-dev \ + libboost-all-dev \ + libcfitsio-dev \ + libhdf5-dev \ + wcslib-dev \ + libatlas-base-dev \ + liblapack-dev \ + python3-tk \ + libreadline6-dev \ + subversion \ + liblog4cplus-dev \ + libhdf5-dev \ + libncurses5-dev \ + libsofa1-dev \ + flex \ + bison \ + libbison-dev \ + # Reference image generation dependencies + make + +RUN apt-get update +RUN apt-get install -y $DEB_SETUP_DEPENDENCIES +RUN apt-get install -y $DEB_DEPENCENDIES + +RUN python3 -m pip install -U pip setuptools wheel + +ENV PATH /usr/local/bin:$PATH +ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH +ENV PYTHONPATH /usr/local/lib/python2.7/site-packages:$PYTHONPATH + +# Latest GCC segfaults when compiling casacore +RUN rm /usr/bin/gcc /usr/bin/g++ /usr/bin/cpp /usr/bin/cc +RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/gcc +RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/g++ +RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/cc +RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/cpp +RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran + +##################################################################### +## BUILD CASACORE FROM SOURCE +##################################################################### +RUN mkdir /src +WORKDIR /src +RUN wget https://github.com/casacore/casacore/archive/v3.3.0.tar.gz +RUN tar xvf v3.3.0.tar.gz +RUN mkdir casacore-3.3.0/build +WORKDIR /src/casacore-3.3.0/build +RUN cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPRECATED=ON -DBUILD_PYTHON=OFF -DBUILD_PYTHON3=ON ../ +RUN make -j 4 +RUN make install +RUN ldconfig +WORKDIR /src +RUN rm v3.3.0.tar.gz +RUN wget https://github.com/casacore/python-casacore/archive/v3.3.0.tar.gz +RUN tar xvf v3.3.0.tar.gz +WORKDIR /src/python-casacore-3.3.0 +RUN python3 -m pip install . +WORKDIR / +RUN python3 -c "from pyrap.tables import table as tbl" + +##################################################################### +## Get CASACORE ephem data +##################################################################### +RUN mkdir -p /usr/share/casacore/data/ +WORKDIR /usr/share/casacore/data/ +RUN apt-get install -y rsync +RUN rsync -avz rsync://casa-rsync.nrao.edu/casa-data . + +##################################################################### +## BUILD MAKEMS FROM SOURCE AND TEST +##################################################################### +RUN mkdir -p /src/ +WORKDIR /src +ENV BUILD /src +RUN wget https://github.com/ska-sa/makems/archive/1.5.3.tar.gz +RUN tar xvf 1.5.3.tar.gz +RUN mkdir -p $BUILD/makems-1.5.3/LOFAR/build/gnu_opt +WORKDIR $BUILD/makems-1.5.3/LOFAR/build/gnu_opt +RUN cmake -DCMAKE_MODULE_PATH:PATH=$BUILD/makems-1.5.3/LOFAR/CMake \ +-DUSE_LOG4CPLUS=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release ../.. +RUN make -j 16 +RUN make install + +ENV PATH=/src/makems-1.5.3/LOFAR/build/gnu_opt/CEP/MS/src:${PATH} +WORKDIR $BUILD/makems-1.5.3/test +RUN makems WSRT_makems.cfg + +##################################################################### +## BUILD CASArest from source +##################################################################### +WORKDIR /src +RUN wget https://github.com/casacore/casarest/archive/v1.7.0.tar.gz +RUN tar xvf v1.7.0.tar.gz +WORKDIR /src/casarest-1.7.0 +RUN mkdir -p build +WORKDIR /src/casarest-1.7.0/build +RUN cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release ../ +RUN make -j 4 +RUN make install +RUN ldconfig + +##################################################################### +## BUILD MEQTREES FROM SOURCE AND TEST +##################################################################### +# Owlcat +WORKDIR /src +RUN wget https://github.com/ska-sa/owlcat/archive/v1.6.0.tar.gz +RUN tar -xvf v1.6.0.tar.gz +WORKDIR /src/owlcat-1.6.0 +RUN python3 -m pip install . + +# kittens +WORKDIR /src +RUN wget https://github.com/ska-sa/kittens/archive/v1.4.3.tar.gz +RUN tar -xvf v1.4.3.tar.gz +WORKDIR /src/kittens-1.4.3 +RUN python3 -m pip install . + +# purr +WORKDIR /src +RUN wget https://github.com/ska-sa/purr/archive/v1.5.0.tar.gz +RUN tar -xvf v1.5.0.tar.gz +WORKDIR /src/purr-1.5.0 +RUN python3 -m pip install . + +# tigger-lsm +WORKDIR /src +RUN rm v1.6.0.tar.gz +RUN wget https://github.com/ska-sa/tigger-lsm/archive/v1.6.0.tar.gz +RUN tar -xvf v1.6.0.tar.gz +WORKDIR /src/tigger-lsm-1.6.0 +RUN python3 -m pip install . + +# Cattery +WORKDIR /src +RUN wget https://github.com/ska-sa/meqtrees-cattery/archive/v1.7.0.tar.gz +RUN tar -xvf v1.7.0.tar.gz.1 +WORKDIR /src/meqtrees-cattery-1.7.0 +RUN python3 -m pip install . + +# blitz +WORKDIR /src +RUN wget https://github.com/blitzpp/blitz/archive/1.0.1.tar.gz +RUN tar -xvf 1.0.1.tar.gz +WORKDIR /src/blitz-1.0.1 +RUN ./configure --enable-shared +RUN make -j16 +RUN make install +RUN ldconfig + +# timba +RUN apt-get install -y libqdbm-dev +WORKDIR /src +RUN wget https://github.com/ska-sa/meqtrees-timba/archive/v1.8.0.tar.gz +RUN tar -xvf v1.8.0.tar.gz +RUN mkdir /src/meqtrees-timba-1.8.0/build +WORKDIR /src/meqtrees-timba-1.8.0/build +RUN cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON_3=ON .. +RUN make -j4 +RUN make install +RUN ldconfig + +# get the test from pyxis +WORKDIR /src +RUN rm v1.7.0.tar.gz +RUN wget https://github.com/ska-sa/pyxis/archive/v1.7.0.tar.gz +RUN tar -xvf v1.7.0.tar.gz +WORKDIR /src/pyxis-1.7.0 +RUN python3 -m pip install . +RUN cp -r /src/pyxis-1.7.0/Pyxis/recipies /usr/local/lib/python3.6/dist-packages/Pyxis/ + +# run test when built +RUN apt-get install -y python3-pyqt4 +RUN python3 -m pip install nose +WORKDIR /usr/local/lib/python3.6/dist-packages/Pyxis/recipies/meqtrees-batch-test +RUN python3 -m "nose" + +##################################################################### +## BUILD LOFAR FROM SOURCE +##################################################################### +WORKDIR /src +RUN git clone -b 'v4.1.1' https://github.com/lofar-astron/LOFARBeam.git +WORKDIR LOFARBeam +RUN mkdir -p build/gnucxx11_opt +WORKDIR build/gnucxx11_opt +RUN cmake -DCMAKE_INSTALL_PREFIX=/usr -DPYTHON_EXECUTABLE=$(which python3) ../../ +RUN make -j16 +RUN make install +WORKDIR / +RUN touch /usr/lib/python3.6/site-packages/lofar/__init__.py +ENV PYTHONPATH "/usr/lib/python3.6/site-packages:$PYTHONPATH" +RUN python3 -c "import lofar.stationresponse as lsr" + +RUN apt-get update +RUN apt-get install -y gfortran + +##################################################################### +## BUILD DDF FROM SOURCE +##################################################################### +#Copy DDFacet and SkyModel into the image +ADD DDFacet /src/DDFacet/DDFacet +ADD SkyModel /src/DDFacet/SkyModel +ADD MANIFEST.in /src/DDFacet/MANIFEST.in +ADD setup.py /src/DDFacet/setup.py +ADD setup.cfg /src/DDFacet/setup.cfg +ADD README.rst /src/DDFacet/README.rst +ADD pyproject.toml /src/DDFacet/pyproject.toml +ADD .git /src/DDFacet/.git +ADD .gitignore /src/DDFacet/.gitignore +ADD .gitmodules /src/DDFacet/.gitmodules + +RUN cd /src/DDFacet/ && git submodule update --init --recursive && cd / +# Finally install DDFacet +RUN rm -rf /src/DDFacet/DDFacet/cbuild +RUN python3 -m pip install -U "/src/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support,kms-support]" + +# Set MeqTrees Cattery path to installation directory +ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/ +ENV PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH + +RUN python3 -c "import Siamese" +RUN python3 -c "import bdsf" + +# perform some basic tests +RUN DDF.py --help +RUN MakeMask.py --help +RUN MakeCatalog.py --help +RUN MakeModel.py --help +RUN MaskDicoModel.py --help +RUN ClusterCat.py --help + +###################################################################### + # Switch to Python 3 environment USER kat ENV PATH="$PATH_PYTHON3" VIRTUAL_ENV="$VIRTUAL_ENV_PYTHON3" @@ -32,3 +299,6 @@ LABEL maintainer="sdpdev+katsdpcontimh@ska.ac.za" COPY --from=build --chown=kat:kat /home/kat/ve3 /home/kat/ve3 ENV PATH="$PATH_PYTHON3" VIRTUAL_ENV="$VIRTUAL_ENV_PYTHON3" + +ENTRYPOINT ["DDF.py"] +CMD ["--help"] diff --git a/docker-compose.yml b/docker-compose.yml index 28e8e57..2696285 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: context: . dockerfile: Dockerfile - container_name: katcontimh + container_name: katsdpcontimh image: sdp-docker-registry.kat.ac.za:5000/katsdpcontimh network_mode: host restart: "no" diff --git a/pre-commit-config.yaml b/pre-commit-config.yaml deleted file mode 100644 index 5edff18..0000000 --- a/pre-commit-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 - hooks: - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace -- repo: https://github.com/psf/black - rev: 21.12b0 - hooks: - - id: black diff --git a/requirements.txt b/requirements.txt index 60438e8..5a719ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,14 @@ -d https://raw.githubusercontent.com/ska-sa/katsdpdockerbase/master/docker-base-gpu-build/requirements.txt astropy +cfvg dask +identify +nodeenv numba numpy<1.21,>=1.18 pre-commit==2.9.2 pytest scipy typing-extensions +virtualenv