Cross Compiling

Issues:

• need cross compile toolchain
  • you need to find or build an entire toolchain environment; ubuntu has some, others have some, Crosstool-NG will build one from scratch
  • for autotools, you typically need to define tools like ar, gcc, g++ in the build script
  • for cmake, you typically need a toolchain file to specify details
• need to cross compile any dependencies (static link what you can)
• need to match the libraries on target machine for any dynamic linking requirements
• need to match the build to the target kernel (or you might get "kernel too old")

libz example

echo Getting source code for libraries... if [ ! -d zlib ]; then wget http://zlib.net/zlib-${zlib_version}.tar.gz untar zlib-${zlib_version}.tar.gz ln -s zlib-${zlib_version} zlib fi echo Building libraries... if [ "$rebuild_libs" = true ]; then export cross=arm-linux-gnueabi- # ZLIB cd zlib ./configure --prefix=${install_root} make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib" make install cd .. fi

openssl example

echo Getting source code for libraries... if [ ! -d openssl ]; then wget https://www.openssl.org/source/openssl-${openssl_version}.tar.gz untar openssl-${openssl_version}.tar.gz ln -s openssl-${openssl_version} openssl fi echo Building libraries... if [ "$rebuild_libs" = true ]; then export cross=arm-linux-gnueabi- # OPENSSL cd openssl ./Configure dist --prefix=${install_root} --openssldir=${install_root} make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib" make install cd .. fi

multi-lib complete example

# Cross compile env for armv4 chipset # Conel cellular modem (Verizon/Pogens) # ARM 926EJ-S processor # Using toolchain on vpn.causam.com that was built a while ago by Bryan G. with Crosstool-NG 1.20.0 # We will cross compile these libraries: # zlib for compression # openssl for encryption - NOTE that it is considered unsafe to bake zlib directly into openssl # libwebsockets for wss # LIB VERSIONS: No reason I know of not to use the latest # It's just that the versions are needed in the script zlib_version=1.2.8 # see http://www.zlib.net/ openssl_version=1.1.0c # see https://www.openssl.org/source/ lws_branch=v2.0-stable # Turn this on at least at the first run, again as needed. rebuild_libs=true # ========================================== # cross compile toolchain magic happens here # ========================================== nickname=armv4 # We will use a cross-compile toolchain that was built a while ago by Bryan G. with Crosstool-NG 1.20.0 # We will build on an older machine (vpn.causam.com) since this did not seem to help limit the kernel: --enable-kernel=${max_kernel} if [ ! -d /opt/crosstool-source-v4 ]; then echo "Cross-compile toolchain not found..." kill -SIGINT $$; fi export cross=arm-926ejs-linux-gnueabi- CC=${cross}gcc CMAKE_TOOLCHAIN_FILE=/home/m/development/causam/git/np/nop-bigress-client-c/${nickname}/toolchain.cmake export PATH=/opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/bin:${PATH} export install_root=/home/m/development/${nickname} # ========================================== echo Getting source code for libraries... cd ${install_root}/source if [ ! -d zlib ]; then wget http://zlib.net/zlib-${zlib_version}.tar.gz tar -xzvf zlib-${zlib_version}.tar.gz ln -s zlib-${zlib_version} zlib fi if [ ! -d openssl ]; then wget https://www.openssl.org/source/openssl-${openssl_version}.tar.gz tar -xzvf openssl-${openssl_version}.tar.gz ln -s openssl-${openssl_version} openssl fi if [ ! -d libwebsockets ]; then git clone https://github.com/warmcat/libwebsockets.git cd libwebsockets git checkout ${lws_branch} cd .. fi echo Building libraries... if [ "$rebuild_libs" = true ]; then # Good help: http://stackoverflow.com/questions/11841919/cross-compile-openssh-for-arm # BUT full of landmines... # Use [file ##.so.##] or [file executable] to find out the architecture of an asset. # ZLIB # THIS IS EXTREMELY BRITTLE, do not adjust # THIS DOES NOT USE 'ar r', IT USES 'ar', BUT OPENSSL WILL USE 'ar r' # Make sure we build static # NOTE the global CC define WILL NOT WORK HERE, use this CC define specifically or it WILL BREAK cd zlib CC="${cross}gcc" AR="${cross}ar" RANLIB="${cross}ranlib" ./configure --static --prefix=${install_root} make clean make make install cd .. # OPENSSL # THIS IS EXTREMELY BRITTLE, do not adjust cd openssl ./Configure dist --prefix=${install_root} --openssldir=${install_root} make clean make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib" make install cd .. # LIBWEBSOCKETS # NOTE: we basically need most lws features, with the exception of PLUGINS support. cd libwebsockets git pull mkdir -p cmake-${nickname}-release cd cmake-${nickname}-release rm -rf CMakeCache.txt CMakeFiles cmake_install.cmake *.cpb Makefile cmake -static \ -DCMAKE_INSTALL_PREFIX:PATH=${install_root} \ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \ -DZLIB_LIBRARY=${install_root}/lib \ -DZLIB_INCLUDE_DIR=${install_root}/include \ -DLWS_OPENSSL_LIBRARIES=${install_root}/lib \ -DLWS_OPENSSL_INCLUDE_DIRS=${install_root}/include \ -DLWS_WITH_SHARED=OFF \ -DLWS_WITHOUT_TESTAPPS=1 \ -DLWS_WITH_SSL=1 \ -DLWS_WITH_ZLIB=1 \ -DLWS_WITHOUT_EXTENSIONS=0 \ -DLWS_HAVE_SSL_CTX_set1_param=1 \ -DLWS_WITHOUT_DAEMONIZE=0 \ -DLWS_WITHOUT_SERVER=1 \ -DLWS_MAX_SMP=4 \ .. make make install cd ../.. fi echo ========================================================= echo Cross-compiled libraries have been built here [${install_root}/lib]: echo ========================================================= ls -la ${install_root}/lib echo ========================================================= echo Copy nop-client source and toolchain CMakeLists.txt... mkdir -p nop-client cp -rp /home/m/development/causam/git/np/nop-bigress-client-c/* nop-client/ cp -rp /home/m/development/causam/git/np/nop-bigress-client-c/${nickname}/CMakeLists.txt nop-client/ cd nop-client mkdir -p cmake-${nickname}-release cd cmake-${nickname}-release echo Building nop-client... rm -rf CMakeCache.txt CMakeFiles cmake_install.cmake *.cpb Makefile cmake -static \ -DCMAKE_INSTALL_PREFIX:PATH=${install_root} \ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \ .. make echo ========================================================= echo Binary details of build... echo ========================================================= file nop-client echo ========================================================= echo Copying to remote device... ssh -p 2222 root@sp-c703.energynet.io "cat > nop-client-${nickname}" < nop-client ssh -p 2222 root@sp-c703.energynet.io "chmod a+x nop-client-${nickname}" # Here are some other remote commands # # rm the bin (had to do this when cp failed due to lack of space on device) # ssh -p 2222 root@sp-c703.energynet.io "rm nop-client-${nickname}" # # cp FROM the device back to host (again, to clean space on device) # ssh -p 2222 root@sp-c703.energynet.io "cat nop-client.0" > nop-client.0

cmake toolchain file example

# https://github.com/zyga/cmake-toolchains/blob/master/Toolchain-Ubuntu-gnueabi.cmake # Sample toolchain file for building for ARM from an Ubuntu Linux system. # # Typical usage: # *) install cross compiler: `sudo apt-get install gcc-arm-linux-gnueabi` # *) cd build # *) cmake -DCMAKE_TOOLCHAIN_FILE=~/Toolchain-Ubuntu-gnueabi.cmake .. set(CMAKE_SYSTEM_NAME Linux) set(TOOLCHAIN_PREFIX arm-926ejs-linux-gnueabi) # cross compilers to use for C and C++ set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) # target environment on the build host system # set 1st to dir with the cross compiler's C/C++ headers/libs set(CMAKE_FIND_ROOT_PATH /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi) # modify default behavior of FIND_XXX() commands to # search for headers/libs in the target environment and # search for programs in the build host environment set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

CMakeLists.txt example

cmake_minimum_required(VERSION 2.8) # MDM silently append relative paths to absolute paths without warnings cmake_policy(SET CMP0015 NEW) # MDM Set the executable name! # project(nop-bigress-client-c) project(nop-client) # ------------------------------- # To ensure we get a static build. # For this to work, the linker needs to be able to find all libraries. # Make sure they are included next in [include|link]_directories. # Also make sure they are included in TARGET_LINK_LIBRARIES. # ------------------------------- SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a") SET(BUILD_SHARED_LIBRARIES OFF) SET(CMAKE_EXE_LINKER_FLAGS "-static") # ------------------------------- # ------------------------------- # MDM For cross compile, we must provide all lib paths. # We have ours: /home/m/development/armv4 # And the toolchain is here: /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi # ------------------------------- include_directories(/home/m/development/armv4/include /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/include) link_directories(/home/m/development/armv4/lib /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/lib) # ------------------------------- # ------------------------------- # MDM Build *everything* in ./src # ------------------------------- # WARNING: [make] won't pick up new files until you rerun [cmake]. # Change this to be explicit if that becomes a problem. # Easy to change: # set( NOP_SOURCES src/nop_client.c src/helpers.c ) aux_source_directory(./src NOP_SOURCES) add_executable( ${PROJECT_NAME} ${NOP_SOURCES} ) # The main linker line used to create the executable # MDM added [c] after ubuntu xenial started giving segfaults in dl -> openssl - didn't help! # see http://stackoverflow.com/questions/7526255/unknown-reference-to-dlopen-in-dlopen # TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets ssl crypto pthread dl z c) # MDM NO LINKING since we built websockets static! no, this doesn't work... #TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets) # MDM cJSON requires floor + pow, in the C math lib, so we add "m" TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets ssl crypto pthread dl z c m)