multi stage build

2020-02-12 15:56:29 +01:00
parent 7254e8900f
commit ae4b053b69
5 changed files with 218 additions and 26 deletions
--- a/62
+++ b/62
@@ -1,10 +1,12 @@
-# Private temp download stage
-FROM ubuntu:18.04 as intermediate
+ARG UI_MODE=cli
+
+# Donwnload stage
+FROM ubuntu:18.04 AS download
 ENV DEBIAN_FRONTEND=noninteractive TZ=Europe/Berlin

-# Setup git
+# Setup deps
 RUN apt-get update && \
-    apt-get -y install git && \
+    apt-get -y install --no-install-recommends git openssh-client ca-certificates && \
    rm -rf /var/lib/apt/lists/*

 # Setup key and server for proprietary code
@@ -28,38 +30,58 @@ RUN git clone https://github.com/MOVE-II/move2radio.git && \
    cd move2radio-prop/ldpc && git checkout 940e8f96ccae831c7d3ad6b9497710696cd7f324 && git reset --hard && cd ../.. && \
    git clone git@${PROP_SERVER}:${PROP_DIR_SDR}/gr-ccsds.git move2radio-prop/gr-ccsds && \
    cd move2radio-prop/gr-ccsds && git checkout 5777e4176fe381327fe6cf55b7ec0dfe3c63b3f6 && git reset --hard && cd ../..
-RUN git clone git@${PROP_SERVER}:${PROP_DIR_SDR}/com_utilities.git move2radio-prop/com_utilities

-# Build stage
-FROM ubuntu:18.04
+
+# Base stage with deps
+FROM ubuntu:18.04 AS base-cli
 ENV DEBIAN_FRONTEND=noninteractive TZ=Europe/Berlin

 # Setup build deps
 RUN apt-get update && \
-    apt-get -y install build-essential git cmake autoconf unzip libtool-bin pkgconf libssl-dev \
-        python2.7 python-pip python-mako socat libcppunit-dev swig uuid-dev libusb-1.0-0-dev \
-        python-mako python-numpy python-wxgtk3.0 python-sphinx python-cheetah libzmq3-dev \
-        libcomedi-dev libqt4-opengl-dev python-qt4 libqwt-dev libsdl1.2-dev python-gtk2 python-lxml python-sip-dev && \
+    apt-get -y install --no-install-recommends build-essential git cmake autoconf automake unzip libtool-bin pkgconf libssl-dev \
+        python2.7 python-pip libcppunit-dev swig uuid-dev libusb-1.0-0-dev \
+        python-mako libzmq3-dev libcomedi-dev python-sip-dev socat && \
    rm -rf /var/lib/apt/lists/*

-# Copy sources from download stage
-COPY --from=intermediate /app /app
+
+# Base stage with GUI deps
+FROM base-cli as base-gui
+
+# Setup grc and gui deps
+RUN apt-get update && \
+    apt-get -y install --no-install-recommends python-numpy python-wxgtk3.0 python-cheetah libzmq3-dev \
+        libqt4-opengl-dev python-qt4 libqwt-dev python-gtk2 python-lxml && \
+    rm -rf /var/lib/apt/lists/*
+
+
+# Build stage
+FROM base-${UI_MODE} AS build
+
+# Copy sources
+COPY --from=download /app /app

 # Build sources
 WORKDIR /app/move2radio
-COPY build/CMakeLists.txt /app/move2radio/CMakeLists.txt
+ARG UI_MODE=cli
+COPY build/CMakeLists.${UI_MODE}.txt /app/move2radio/CMakeLists.txt
 RUN mkdir build && cd build && \
    cmake -DAPPIMAGE=OFF -DCONDA_ENV=OFF -DLDPC_OFFLINE=ON -DGR_CCSDS_OFFLINE=ON -DOFFLINE_DEPENDENCY_PATH=/app/move2radio-prop .. && \ 
    make move2radio

-# Install binaries
-COPY build/blocks/* /app/move2radio/build/prefix/usr/share/gnuradio/grc/blocks/
-RUN chmod 644 /app/move2radio/build/prefix/usr/share/gnuradio/grc/blocks/*.xml && \
-    cp -a build/prefix/usr/. /usr/ && \
-    ldconfig

-# Entrypoint
+# Run stage
+FROM base-${UI_MODE} AS run
+
+# Copy sources from build stage
+COPY --from=build /app/move2radio/build/prefix/usr /usr
+
+# Install binaries
+COPY build/blocks/* /usr/share/gnuradio/grc/blocks/
+RUN chmod 644 /usr/share/gnuradio/grc/blocks/*.xml && \
+    ldconfig
 COPY build/fakeserver/* /app/fakeserver/
 COPY build/run.sh /app/run.sh
+
+# Entrypoint
 RUN chmod +x /app/run.sh
 ENTRYPOINT ["/app/run.sh"]
--- a/build/CMakeLists.cli.txt
+++ b/build/CMakeLists.cli.txt
@@ -0,0 +1,166 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.0)
+PROJECT(move2radio
+  LANGUAGES C
+  DESCRIPTION "Portable Version of the MOVE-II groundstation")
+
+INCLUDE(ExternalProject)
+
+IF(UNIX AND NOT APPLE)
+  SET(LINUX TRUE)
+ENDIF()
+
+INCLUDE(cmake/options.cmake)
+INCLUDE(cmake/system_deps.cmake)
+
+INCLUDE(cmake/build_autotools.cmake)
+INCLUDE(cmake/build_cmake.cmake)
+INCLUDE(cmake/build_meson.cmake)
+
+SET(DEPS_PATH "${CMAKE_CURRENT_BINARY_DIR}/deps")
+SET(BUILD_PATH "${CMAKE_CURRENT_BINARY_DIR}/build")
+SET(PREFIX_PATH "${CMAKE_CURRENT_BINARY_DIR}/prefix")
+FILE(MAKE_DIRECTORY ${DEPS_PATH}/src)
+
+SET(ENV{PKG_CONFIG_PATH} ${PREFIX_PATH}/usr/lib/pkgconfig)
+SET(PKG_CONFIG_PATH ENV{PKG_CONFIG_PATH})
+## Custom target deps makes sure all sources are there
+ADD_CUSTOM_TARGET(deps)
+IF(NOT ${NO_DOWNLOAD})
+  ADD_CUSTOM_TARGET(download)
+  ADD_DEPENDENCIES(deps download)
+ENDIF()
+
+## move2deps.cmake contains all sources
+INCLUDE(cmake/dependency.cmake)
+INCLUDE(cmake/move2deps.cmake)
+
+ADD_CUSTOM_TARGET(delete_prefix
+  ${CMAKE_COMMAND} -E remove_directory ${PREFIX_PATH})
+ADD_CUSTOM_TARGET(prefix
+  ${CMAKE_COMMAND} -E make_directory ${PREFIX_PATH}/usr/share
+  COMMAND echo \"CFLAGS=-I${PREFIX_PATH}/usr/include\\nLDFLAGS=\\\"-L${PREFIX_PATH}/usr/lib -Wl,-rpath=${PREFIX_PATH}/usr/lib\\\"\\n\" > ${PREFIX_PATH}/config.site
+  COMMAND rm -f ${PREFIX_PATH}/usr/lib/libuuid.*
+  COMMAND find / -not -path \"${CMAKE_CURRENT_BINARY_DIR}/*\" -name \"libuuid*.so*\" -exec ln -sf {} ${PREFIX_PATH}/usr/lib/libuuid.so \\\\; 2>/dev/null || [ -e ${PREFIX_PATH}/usr/lib/libuuid.so ])
+
+### Anaconda deploys a portable python + pygtk,numpy,pyqt ###
+INCLUDE(cmake/anaconda.cmake)
+
+INCLUDE(ProcessorCount)
+PROCESSORCOUNT(N)
+FIND_PROGRAM(MAKE make)
+FIND_PROGRAM(MESON meson)
+FIND_PROGRAM(NINJA ninja)
+
+### Build all sources ###
+
+# Boost and ldpc Build is more complicated. Moved to own file
+INCLUDE(cmake/boost.cmake)
+INCLUDE(cmake/ldpc.cmake)
+
+MOVEII_BUILD_AUTOTOOLS(fftw EXTRA_OPTIONS --enable-threads --enable-float)
+MOVEII_BUILD_AUTOTOOLS(gsl)
+MOVEII_BUILD_AUTOTOOLS(libiconv)
+MOVEII_BUILD_AUTOTOOLS(rtl-sdr)
+MOVEII_BUILD_AUTOTOOLS(gmp)
+MOVEII_BUILD_AUTOTOOLS(rxvt-unicode EXTRA_OPTIONS --disable-perl --enable-xft CONFIG_SITE ON)
+MOVEII_BUILD_AUTOTOOLS(ntl NO_DEFAULT ON SOURCE_SUFFIX "/src" COPY_SOURCE ON
+  EXTRA_OPTIONS NTL_GMP_LIP=on SHARED=on NATIVE=off DEF_PREFIX=${PREFIX_PATH}/usr)
+MOVEII_BUILD_AUTOTOOLS(ncurses)
+MOVEII_BUILD_CMAKE(airspy)
+MOVEII_BUILD_CMAKE(gr-osmosdr)
+MOVEII_BUILD_CMAKE(curl EXTRA_OPTIONS -DCMAKE_USE_OPENSSL=ON)
+MOVEII_BUILD_CMAKE(curlpp)
+MOVEII_BUILD_CMAKE(gr-ccsds)
+MOVEII_BUILD_CMAKE(gr-iqbal)
+MOVEII_BUILD_CMAKE(libosmosdr)
+MOVEII_BUILD_CMAKE(bladerf EXTRA_OPTIONS -DINSTALL_UDEV_RULES=OFF)
+MOVEII_BUILD_CMAKE(libmirisdr)
+MOVEII_BUILD_CMAKE(soapysdr)
+MOVEII_BUILD_CMAKE(hackrf SOURCE_SUFFIX "/host/")
+MOVEII_BUILD_CMAKE(gr-fcdproplus)
+MOVEII_BUILD_CMAKE(freesrp)
+MOVEII_BUILD_CMAKE(libuhd EXTRA_OPTIONS -DENABLE_E100=ON -DENABLE_E300=ON -DENABLE_EXAMPLES=OFF -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF SOURCE_SUFFIX "/host")
+
+MOVEII_BUILD_CMAKE(gnuradio EXTRA_OPTIONS
+  -DENABLE_INTERNAL_VOLK=ON
+  -DENABLE_DEFAULT=OFF
+  -DENABLE_VOLK=ON
+  -DENABLE_GNURADIO_RUNTIME=ON
+  -DENABLE_PYTHON=ON
+  -DENABLE_GR_AUDIO=ON
+  -DENABLE_GR_BLOCKS=ON
+  -DENABLE_GR_FILTER=ON
+  -DENABLE_GR_FFT=ON
+  -DENABLE_GR_UHD=OFF
+  -DENABLE_GR_FCD=OFF
+  -DENABLE_GR_ANALOG=ON
+  -DENABLE_GR_WAVELET=ON
+  -DENABLE_GRC=OFF
+  -DENABLE_GR_QTGUI=OFF
+  -DENABLE_GR_DIGITAL=ON)
+ADD_DEPENDENCIES(build_gnuradio build_boost build_gsl build_fftw build_libiconv)
+
+MOVEII_BUILD_AUTOTOOLS(fec NO_AUTORECONF ON NO_INSTALL ON)
+## fec is to dumb to properly install:
+ADD_CUSTOM_COMMAND(OUTPUT ${PREFIX_PATH}/usr/lib/libfec.so
+  COMMAND ${CMAKE_COMMAND} -E copy ${FEC_SOURCE}/fec.h ${PREFIX_PATH}/usr/include/fec.h
+  COMMAND ${CMAKE_COMMAND} -E copy ${BUILD_PATH}/fec/libfec.a ${PREFIX_PATH}/usr/lib/libfec.a
+  COMMAND ${CMAKE_COMMAND} -E copy ${BUILD_PATH}/fec/libfec.so ${PREFIX_PATH}/usr/lib/libfec.so
+  DEPENDS build_fec
+  )
+ADD_CUSTOM_TARGET(install_fec
+  DEPENDS ${PREFIX_PATH}/usr/lib/libfec.so)
+
+## Get the dependencies between the sources right:
+ADD_DEPENDENCIES(build_gr_osmosdr build_rtl_sdr build_gnuradio)
+ADD_DEPENDENCIES(build_ntl build_gmp)
+ADD_DEPENDENCIES(build_curlpp build_curl)
+ADD_DEPENDENCIES(install_fec prefix)
+ADD_DEPENDENCIES(build_gr_ccsds install_fec build_gnuradio build_ldpc build_curlpp)
+ADD_DEPENDENCIES(build_gr_fcdproplus build_gnuradio)
+
+ADD_CUSTOM_TARGET(move2radio)
+
+ADD_DEPENDENCIES(move2radio build_gr_ccsds build_gr_osmosdr)
+ADD_DEPENDENCIES(build_libuhd build_boost build_ncurses)
+
+ADD_DEPENDENCIES(build_hackrf build_fftw)
+ADD_DEPENDENCIES(build_freesrp build_boost)
+ADD_DEPENDENCIES(build_rxvt_unicode build_ncurses)
+
+
+
+IF(APPIMAGE)
+  ADD_EXECUTABLE(AppRun AppDir/AppRun.c)
+  SET(APPDIR ${CMAKE_CURRENT_BINARY_DIR}/AppDir)
+  FILE(MAKE_DIRECTORY ${APPDIR})
+  IF(EXISTS ${APPDIR})
+	FILE(REMOVE_RECURSE ${APPDIR})
+  ENDIF()
+  ADD_CUSTOM_COMMAND(OUTPUT ${APPDIR}/AppRun
+	COMMAND cp -r ${PREFIX_PATH} ${APPDIR}
+	COMMAND ${CMAKE_COMMAND} -E remove_directory ${APPDIR}/usr/include
+	COMMAND ${CMAKE_COMMAND} -E remove_directory ${APPDIR}/usr/pkgs
+	COMMAND ${CMAKE_COMMAND} -E remove_directory ${APPDIR}/usr/ssl
+	COMMAND rm -rf ${APPDIR}/usr/lib/libuuid.so*
+	COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/AppDir ${APPDIR}
+	COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/AppRun ${APPDIR}/AppRun
+	DEPENDS AppRun move2radio build_rxvt_unicode)
+
+  ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/squashfs-root/AppRun
+	COMMAND chmod +x ${DEPS_PATH}/src/appimagetool-x86_64.AppImage
+	COMMAND ${DEPS_PATH}/src/appimagetool-x86_64.AppImage --appimage-extract
+	DEPENDS appimagetool)
+
+  ADD_CUSTOM_TARGET(appimagetool_extract
+	DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/squashfs-root/AppRun)
+
+  ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/Gnuradio_Companion-x86_64.AppImage
+	COMMAND env ARCH=x86_64 ${CMAKE_CURRENT_BINARY_DIR}/squashfs-root/AppRun ${APPDIR}
+	DEPENDS appimagetool_extract ${APPDIR}/AppRun)
+
+  ADD_CUSTOM_TARGET(appimage_build
+	DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/Gnuradio_Companion-x86_64.AppImage)
+
+  ADD_DEPENDENCIES(appimage appimage_build)
+ENDIF()
--- a/build/CMakeLists.gui.txt
+++ b/build/CMakeLists.gui.txt
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -4,6 +4,7 @@ cd "${0%/*}/.."
 source ./config/config.sh
 export PROP_SERVER_KEY=$(cat ./config/id_rsa)

-docker build \
+export UI_MODE=cli
+docker build --build-arg UI_MODE \
    --build-arg PROP_SERVER --build-arg PROP_SERVER_USER --build-arg PROP_SERVER_KEY \
-    --build-arg PROP_DIR_SDR --build-arg PROP_DIR_DECODER -t registry.chrz.de:443/move2docker:latest-$(arch) .
+    --build-arg PROP_DIR_SDR --build-arg PROP_DIR_DECODER -t registry.chrz.de:443/move2docker:$UI_MODE-latest-$(arch) .
--- a/scripts/push.sh
+++ b/scripts/push.sh
@@ -1,8 +1,11 @@
 #!/bin/bash

+docker tag registry.chrz.de:443/move2docker:cli-latest-$(arch) registry.chrz.de:443/move2docker:latest-$(arch)
+docker push registry.chrz.de:443/move2docker:cli-latest-$(arch)
+docker push registry.chrz.de:443/move2docker:gui-latest-$(arch)
 docker push registry.chrz.de:443/move2docker:latest-$(arch)

-docker manifest create registry.chrz.de:443/move2docker:latest registry.chrz.de:443/move2docker:latest-x86_64 registry.chrz.de:443/move2docker:latest-armv7l
-docker manifest annotate registry.chrz.de:443/move2docker:latest registry.chrz.de:443/move2docker:latest-x86_64 --os linux --arch amd64
-docker manifest annotate registry.chrz.de:443/move2docker:latest registry.chrz.de:443/move2docker:latest-armv7l --os linux --arch armv7
-docker manifest push registry.chrz.de:443/move2docker:latest --purge
+# docker manifest create registry.chrz.de:443/move2docker:latest registry.chrz.de:443/move2docker:latest-x86_64 registry.chrz.de:443/move2docker:latest-armv7l
+# docker manifest annotate registry.chrz.de:443/move2docker:latest registry.chrz.de:443/move2docker:latest-x86_64 --os linux --arch amd64
+# docker manifest annotate registry.chrz.de:443/move2docker:latest registry.chrz.de:443/move2docker:latest-armv7l --os linux --arch armv7
+# docker manifest push registry.chrz.de:443/move2docker:latest --purge