Cross Compiling

From Bitpost wiki

Issues:

  • need cross compile toolchain
    • you need to find or build an entire toolchain environment; ubuntu has some, others have some, Crosstool-NG will build one from scratch
    • for autotools, you typically need to define tools like ar, gcc, g++ in the build script
    • for cmake, you typically need a toolchain file to specify details
  • need to cross compile any dependencies (static link what you can)
  • need to match the libraries on target machine for any dynamic linking requirements
  • need to match the build to the target kernel (or you might get "kernel too old")
libz example
   echo Getting source code for libraries...
   if [ ! -d zlib ]; then
       wget http://zlib.net/zlib-${zlib_version}.tar.gz
       untar zlib-${zlib_version}.tar.gz
       ln -s zlib-${zlib_version} zlib
   fi
   echo Building libraries...
   if [ "$rebuild_libs" = true ]; then
   
       export cross=arm-linux-gnueabi-
   
       # ZLIB
       cd zlib
       ./configure --prefix=${install_root}
       make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib"
       make install
       cd ..
   fi
openssl example
   echo Getting source code for libraries...
   if [ ! -d openssl ]; then
       wget https://www.openssl.org/source/openssl-${openssl_version}.tar.gz
       untar openssl-${openssl_version}.tar.gz
       ln -s openssl-${openssl_version} openssl
   fi
   
   echo Building libraries...
   
   if [ "$rebuild_libs" = true ]; then
   
       export cross=arm-linux-gnueabi-
   
       # OPENSSL
       cd openssl
       ./Configure dist --prefix=${install_root} --openssldir=${install_root}
       make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib"
       make install
       cd ..
   fi
multi-lib complete example
   # Cross compile env for armv4 chipset 
   # Conel cellular modem (Verizon/Pogens)
   # ARM 926EJ-S processor
   # Using toolchain on vpn.causam.com that was built a while ago by Bryan G. with Crosstool-NG 1.20.0
   
   # We will cross compile these libraries:
   # zlib for compression
   # openssl for encryption - NOTE that it is considered unsafe to bake zlib directly into openssl
   # libwebsockets for wss
   
   
   # LIB VERSIONS: No reason I know of not to use the latest
   # It's just that the versions are needed in the script
   zlib_version=1.2.8     # see http://www.zlib.net/
   openssl_version=1.1.0c # see https://www.openssl.org/source/
   lws_branch=v2.0-stable
   
   
   # Turn this on at least at the first run, again as needed.
   rebuild_libs=true
   
   
   # ==========================================
   # cross compile toolchain magic happens here
   # ==========================================
   
   nickname=armv4
   
   # We will use a cross-compile toolchain that was built a while ago by Bryan G. with Crosstool-NG 1.20.0
   # We will build on an older machine (vpn.causam.com) since this did not seem to help limit the kernel: --enable-kernel=${max_kernel}
   if [ ! -d /opt/crosstool-source-v4 ]; then
       echo "Cross-compile toolchain not found..."
       kill -SIGINT $$;
   fi
   
   export cross=arm-926ejs-linux-gnueabi-
   CC=${cross}gcc
   CMAKE_TOOLCHAIN_FILE=/home/m/development/causam/git/np/nop-bigress-client-c/${nickname}/toolchain.cmake
   export PATH=/opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/bin:${PATH}
   
   export install_root=/home/m/development/${nickname}
   
   # ==========================================
   
   
   echo Getting source code for libraries...
   cd ${install_root}/source
   if [ ! -d zlib ]; then
       wget http://zlib.net/zlib-${zlib_version}.tar.gz
       tar -xzvf zlib-${zlib_version}.tar.gz
       ln -s zlib-${zlib_version} zlib
   fi
   if [ ! -d openssl ]; then
       wget https://www.openssl.org/source/openssl-${openssl_version}.tar.gz
       tar -xzvf openssl-${openssl_version}.tar.gz
       ln -s openssl-${openssl_version} openssl
   fi
   if [ ! -d libwebsockets ]; then
       git clone https://github.com/warmcat/libwebsockets.git
       cd libwebsockets
       git checkout ${lws_branch}
       cd ..
   fi
   
   echo Building libraries...
   
   if [ "$rebuild_libs" = true ]; then
   
       # Good help: http://stackoverflow.com/questions/11841919/cross-compile-openssh-for-arm
       # BUT full of landmines...    
       # Use [file ##.so.##] or [file executable] to find out the architecture of an asset.
   
       # ZLIB
       # THIS IS EXTREMELY BRITTLE, do not adjust
       # THIS DOES NOT USE 'ar r', IT USES 'ar', BUT OPENSSL WILL USE 'ar r'
       # Make sure we build static
       # NOTE the global CC define WILL NOT WORK HERE, use this CC define specifically or it WILL BREAK
       cd zlib
       CC="${cross}gcc" AR="${cross}ar" RANLIB="${cross}ranlib" ./configure --static --prefix=${install_root}
   
       make clean
       make
       make install
       cd ..
       
       # OPENSSL
       # THIS IS EXTREMELY BRITTLE, do not adjust
       cd openssl
       ./Configure dist --prefix=${install_root} --openssldir=${install_root}
       make clean
       make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib"
       make install
       cd ..
   
       # LIBWEBSOCKETS
       # NOTE: we basically need most lws features, with the exception of PLUGINS support.
       cd libwebsockets
       git pull
       mkdir -p cmake-${nickname}-release
       cd cmake-${nickname}-release
       rm -rf CMakeCache.txt CMakeFiles cmake_install.cmake *.cpb Makefile
       cmake -static \
           -DCMAKE_INSTALL_PREFIX:PATH=${install_root}         \
           -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}      \
           -DZLIB_LIBRARY=${install_root}/lib                  \
           -DZLIB_INCLUDE_DIR=${install_root}/include          \
           -DLWS_OPENSSL_LIBRARIES=${install_root}/lib         \
           -DLWS_OPENSSL_INCLUDE_DIRS=${install_root}/include  \
           -DLWS_WITH_SHARED=OFF           \
           -DLWS_WITHOUT_TESTAPPS=1        \
           -DLWS_WITH_SSL=1                \
           -DLWS_WITH_ZLIB=1               \
           -DLWS_WITHOUT_EXTENSIONS=0      \
           -DLWS_HAVE_SSL_CTX_set1_param=1 \
           -DLWS_WITHOUT_DAEMONIZE=0       \
           -DLWS_WITHOUT_SERVER=1          \
           -DLWS_MAX_SMP=4                 \
           ..
       make
       make install
       cd ../..
   fi
   
   echo =========================================================
   echo Cross-compiled libraries have been built here [${install_root}/lib]:
   echo =========================================================
   ls -la ${install_root}/lib
   echo =========================================================
   
   echo Copy nop-client source and toolchain CMakeLists.txt...
   mkdir -p nop-client
   cp -rp /home/m/development/causam/git/np/nop-bigress-client-c/* nop-client/
   cp -rp /home/m/development/causam/git/np/nop-bigress-client-c/${nickname}/CMakeLists.txt nop-client/
   cd nop-client
   mkdir -p cmake-${nickname}-release
   cd cmake-${nickname}-release
   
   echo Building nop-client...
   rm -rf CMakeCache.txt CMakeFiles cmake_install.cmake *.cpb Makefile
   cmake -static \
       -DCMAKE_INSTALL_PREFIX:PATH=${install_root}         \
       -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}      \
       ..
   make
   
   echo =========================================================
   echo Binary details of build...
   echo =========================================================
   file nop-client
   echo =========================================================
   
   echo Copying to remote device...
   ssh -p 2222 root@sp-c703.energynet.io "cat > nop-client-${nickname}" < nop-client
   ssh -p 2222 root@sp-c703.energynet.io "chmod a+x nop-client-${nickname}"
   
   # Here are some other remote commands
   #
   # rm the bin (had to do this when cp failed due to lack of space on device)
   # ssh -p 2222 root@sp-c703.energynet.io "rm nop-client-${nickname}"
   #
   # cp FROM the device back to host (again, to clean space on device)
   # ssh -p 2222 root@sp-c703.energynet.io "cat nop-client.0" > nop-client.0
cmake toolchain file example
   # https://github.com/zyga/cmake-toolchains/blob/master/Toolchain-Ubuntu-gnueabi.cmake
   # Sample toolchain file for building for ARM from an Ubuntu Linux system.
   #
   # Typical usage:
   #    *) install cross compiler: `sudo apt-get install gcc-arm-linux-gnueabi`
   #    *) cd build
   #    *) cmake -DCMAKE_TOOLCHAIN_FILE=~/Toolchain-Ubuntu-gnueabi.cmake ..
   
   set(CMAKE_SYSTEM_NAME Linux)
   set(TOOLCHAIN_PREFIX arm-926ejs-linux-gnueabi)
   
   # cross compilers to use for C and C++
   set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc)
   set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++)
   
   # target environment on the build host system
   #   set 1st to dir with the cross compiler's C/C++ headers/libs
   set(CMAKE_FIND_ROOT_PATH /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi)
   
   # modify default behavior of FIND_XXX() commands to
   # search for headers/libs in the target environment and
   # search for programs in the build host environment
   set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
   set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
   set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
   
CMakeLists.txt example
   cmake_minimum_required(VERSION 2.8)
   # MDM silently append relative paths to absolute paths without warnings
   cmake_policy(SET CMP0015 NEW)
   # MDM Set the executable name!
   # project(nop-bigress-client-c)
   project(nop-client)
   # -------------------------------
   # To ensure we get a static build.
   # For this to work, the linker needs to be able to find all libraries.
   # Make sure they are included next in [include|link]_directories.
   # Also make sure they are included in TARGET_LINK_LIBRARIES.
   # -------------------------------
   SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
   SET(BUILD_SHARED_LIBRARIES OFF)
   SET(CMAKE_EXE_LINKER_FLAGS "-static")
   # -------------------------------
   # -------------------------------
   # MDM For cross compile, we must provide all lib paths.
   # We have ours: /home/m/development/armv4
   # And the toolchain is here: /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi
   # -------------------------------
   include_directories(/home/m/development/armv4/include /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/include)
   link_directories(/home/m/development/armv4/lib /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/lib)
   # -------------------------------
   # -------------------------------
   # MDM Build *everything* in ./src
   # -------------------------------
   # WARNING: [make] won't pick up new files until you rerun [cmake].
   # Change this to be explicit if that becomes a problem.
   # Easy to change:
   #    set( NOP_SOURCES src/nop_client.c src/helpers.c )
   aux_source_directory(./src NOP_SOURCES)
   add_executable(
       ${PROJECT_NAME}
       ${NOP_SOURCES}
   )
   # The main linker line used to create the executable
   # MDM added [c] after ubuntu xenial started giving segfaults in dl -> openssl - didn't help!
   # see http://stackoverflow.com/questions/7526255/unknown-reference-to-dlopen-in-dlopen
   # TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets ssl crypto pthread dl z c)
   # MDM NO LINKING since we built websockets static!  no, this doesn't work...
   #TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets)
   # MDM cJSON requires floor + pow, in the C math lib, so we add "m"
   TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets ssl crypto pthread dl z c m)