Cross Compiling: Difference between revisions

From Bitpost wiki
No edit summary
No edit summary
 
(4 intermediate revisions by the same user not shown)
Line 1: Line 1:
Issues:
Issues:
* need cross compile toolchain
* need cross compile toolchain
** you need to find or build an entire toolchain environment; ubuntu has some, others have some, Crosstool-NG will build one from scratch
** for autotools, you typically need to define tools like ar, gcc, g++ in the build script
** for cmake, you typically need a toolchain file to specify details
* need to cross compile any dependencies (static link what you can)
* need to cross compile any dependencies (static link what you can)
* need to match the libraries on target machine for any dynamic linking requirements
* need to match the build to the target kernel (or you might get "kernel too old")
{| class="mw-collapsible mw-collapsed wikitable"
{| class="mw-collapsible mw-collapsed wikitable"
! libz example
! libz example
Line 25: Line 31:
     fi
     fi
|}
|}
** {| class="mw-collapsible mw-collapsed wikitable"
{| class="mw-collapsible mw-collapsed wikitable"
! openssl example
! openssl example
|-
|-
Line 49: Line 55:
         cd ..
         cd ..
     fi
     fi
** libwebsockets example
|}
* need to match the libraries on target machine for any dynamic linking requirements
{| class="mw-collapsible mw-collapsed wikitable"
* need to match the build to the target kernel (or you might get "kernel too old")
! multi-lib complete example
|-
|
    # Cross compile env for armv4 chipset
    # Conel cellular modem (Verizon/Pogens)
    # ARM 926EJ-S processor
    # Using toolchain on vpn.causam.com that was built a while ago by Bryan G. with Crosstool-NG 1.20.0
   
    # We will cross compile these libraries:
    # zlib for compression
    # openssl for encryption - NOTE that it is considered unsafe to bake zlib directly into openssl
    # libwebsockets for wss
   
   
    # LIB VERSIONS: No reason I know of not to use the latest
    # It's just that the versions are needed in the script
    zlib_version=1.2.8    # see http://www.zlib.net/
    openssl_version=1.1.0c # see https://www.openssl.org/source/
    lws_branch=v2.0-stable
   
   
    # Turn this on at least at the first run, again as needed.
    rebuild_libs=true
   
   
    # ==========================================
    # cross compile toolchain magic happens here
    # ==========================================
   
    nickname=armv4
   
    # We will use a cross-compile toolchain that was built a while ago by Bryan G. with Crosstool-NG 1.20.0
    # We will build on an older machine (vpn.causam.com) since this did not seem to help limit the kernel: --enable-kernel=${max_kernel}
    if [ ! -d /opt/crosstool-source-v4 ]; then
        echo "Cross-compile toolchain not found..."
        kill -SIGINT $$;
    fi
   
    export cross=arm-926ejs-linux-gnueabi-
    CC=${cross}gcc
    CMAKE_TOOLCHAIN_FILE=/home/m/development/causam/git/np/nop-bigress-client-c/${nickname}/toolchain.cmake
    export PATH=/opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/bin:${PATH}
   
    export install_root=/home/m/development/${nickname}
   
    # ==========================================
   
   
    echo Getting source code for libraries...
    cd ${install_root}/source
    if [ ! -d zlib ]; then
        wget http://zlib.net/zlib-${zlib_version}.tar.gz
        tar -xzvf zlib-${zlib_version}.tar.gz
        ln -s zlib-${zlib_version} zlib
    fi
    if [ ! -d openssl ]; then
        wget https://www.openssl.org/source/openssl-${openssl_version}.tar.gz
        tar -xzvf openssl-${openssl_version}.tar.gz
        ln -s openssl-${openssl_version} openssl
    fi
    if [ ! -d libwebsockets ]; then
        git clone https://github.com/warmcat/libwebsockets.git
        cd libwebsockets
        git checkout ${lws_branch}
        cd ..
    fi
   
    echo Building libraries...
   
    if [ "$rebuild_libs" = true ]; then
   
        # Good help: http://stackoverflow.com/questions/11841919/cross-compile-openssh-for-arm
        # BUT full of landmines...   
        # Use [file ##.so.##] or [file executable] to find out the architecture of an asset.
   
        # ZLIB
        # THIS IS EXTREMELY BRITTLE, do not adjust
        # THIS DOES NOT USE 'ar r', IT USES 'ar', BUT OPENSSL WILL USE 'ar r'
        # Make sure we build static
        # NOTE the global CC define WILL NOT WORK HERE, use this CC define specifically or it WILL BREAK
        cd zlib
        CC="${cross}gcc" AR="${cross}ar" RANLIB="${cross}ranlib" ./configure --static --prefix=${install_root}
   
        make clean
        make
        make install
        cd ..
       
        # OPENSSL
        # THIS IS EXTREMELY BRITTLE, do not adjust
        cd openssl
        ./Configure dist --prefix=${install_root} --openssldir=${install_root}
        make clean
        make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib"
        make install
        cd ..
   
        # LIBWEBSOCKETS
        # NOTE: we basically need most lws features, with the exception of PLUGINS support.
        cd libwebsockets
        git pull
        mkdir -p cmake-${nickname}-release
        cd cmake-${nickname}-release
        rm -rf CMakeCache.txt CMakeFiles cmake_install.cmake *.cpb Makefile
        cmake -static \
            -DCMAKE_INSTALL_PREFIX:PATH=${install_root}        \
            -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}      \
            -DZLIB_LIBRARY=${install_root}/lib                  \
            -DZLIB_INCLUDE_DIR=${install_root}/include          \
            -DLWS_OPENSSL_LIBRARIES=${install_root}/lib        \
            -DLWS_OPENSSL_INCLUDE_DIRS=${install_root}/include  \
            -DLWS_WITH_SHARED=OFF          \
            -DLWS_WITHOUT_TESTAPPS=1        \
            -DLWS_WITH_SSL=1                \
            -DLWS_WITH_ZLIB=1              \
            -DLWS_WITHOUT_EXTENSIONS=0      \
            -DLWS_HAVE_SSL_CTX_set1_param=1 \
            -DLWS_WITHOUT_DAEMONIZE=0      \
            -DLWS_WITHOUT_SERVER=1          \
            -DLWS_MAX_SMP=4                \
            ..
        make
        make install
        cd ../..
    fi
   
    echo =========================================================
    echo Cross-compiled libraries have been built here [${install_root}/lib]:
    echo =========================================================
    ls -la ${install_root}/lib
    echo =========================================================
   
    echo Copy nop-client source and toolchain CMakeLists.txt...
    mkdir -p nop-client
    cp -rp /home/m/development/causam/git/np/nop-bigress-client-c/* nop-client/
    cp -rp /home/m/development/causam/git/np/nop-bigress-client-c/${nickname}/CMakeLists.txt nop-client/
    cd nop-client
    mkdir -p cmake-${nickname}-release
    cd cmake-${nickname}-release
   
    echo Building nop-client...
    rm -rf CMakeCache.txt CMakeFiles cmake_install.cmake *.cpb Makefile
    cmake -static \
        -DCMAKE_INSTALL_PREFIX:PATH=${install_root}        \
        -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}      \
        ..
    make
   
    echo =========================================================
    echo Binary details of build...
    echo =========================================================
    file nop-client
    echo =========================================================
   
    echo Copying to remote device...
    ssh -p 2222 root@sp-c703.energynet.io "cat > nop-client-${nickname}" < nop-client
    ssh -p 2222 root@sp-c703.energynet.io "chmod a+x nop-client-${nickname}"
   
    # Here are some other remote commands
    #
    # rm the bin (had to do this when cp failed due to lack of space on device)
    # ssh -p 2222 root@sp-c703.energynet.io "rm nop-client-${nickname}"
    #
    # cp FROM the device back to host (again, to clean space on device)
    # ssh -p 2222 root@sp-c703.energynet.io "cat nop-client.0" > nop-client.0
|}
{| class="mw-collapsible mw-collapsed wikitable"
! cmake toolchain file example
|-
|
    # https://github.com/zyga/cmake-toolchains/blob/master/Toolchain-Ubuntu-gnueabi.cmake
    # Sample toolchain file for building for ARM from an Ubuntu Linux system.
    #
    # Typical usage:
    #    *) install cross compiler: `sudo apt-get install gcc-arm-linux-gnueabi`
    #    *) cd build
    #    *) cmake -DCMAKE_TOOLCHAIN_FILE=~/Toolchain-Ubuntu-gnueabi.cmake ..
   
    set(CMAKE_SYSTEM_NAME Linux)
    set(TOOLCHAIN_PREFIX arm-926ejs-linux-gnueabi)
   
    # cross compilers to use for C and C++
    set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc)
    set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++)
   
    # target environment on the build host system
    #  set 1st to dir with the cross compiler's C/C++ headers/libs
    set(CMAKE_FIND_ROOT_PATH /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi)
   
    # modify default behavior of FIND_XXX() commands to
    # search for headers/libs in the target environment and
    # search for programs in the build host environment
    set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
    set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
    set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
   
|}
{| class="mw-collapsible mw-collapsed wikitable"
! CMakeLists.txt example
|-
|
    cmake_minimum_required(VERSION 2.8)
 
    # MDM silently append relative paths to absolute paths without warnings
    cmake_policy(SET CMP0015 NEW)
 
    # MDM Set the executable name!
    # project(nop-bigress-client-c)
    project(nop-client)
 
    # -------------------------------
    # To ensure we get a static build.
    # For this to work, the linker needs to be able to find all libraries.
    # Make sure they are included next in [include|link]_directories.
    # Also make sure they are included in TARGET_LINK_LIBRARIES.
    # -------------------------------
    SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
    SET(BUILD_SHARED_LIBRARIES OFF)
    SET(CMAKE_EXE_LINKER_FLAGS "-static")
    # -------------------------------
 
    # -------------------------------
    # MDM For cross compile, we must provide all lib paths.
    # We have ours: /home/m/development/armv4
    # And the toolchain is here: /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi
    # -------------------------------
    include_directories(/home/m/development/armv4/include /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/include)
    link_directories(/home/m/development/armv4/lib /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/lib)
    # -------------------------------
 
    # -------------------------------
    # MDM Build *everything* in ./src
    # -------------------------------
    # WARNING: [make] won't pick up new files until you rerun [cmake].
    # Change this to be explicit if that becomes a problem.
    # Easy to change:
    #    set( NOP_SOURCES src/nop_client.c src/helpers.c )
    aux_source_directory(./src NOP_SOURCES)
 
    add_executable(
        ${PROJECT_NAME}
        ${NOP_SOURCES}
    )
 
    # The main linker line used to create the executable
    # MDM added [c] after ubuntu xenial started giving segfaults in dl -> openssl - didn't help!
    # see http://stackoverflow.com/questions/7526255/unknown-reference-to-dlopen-in-dlopen
    # TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets ssl crypto pthread dl z c)
    # MDM NO LINKING since we built websockets static!  no, this doesn't work...
    #TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets)
    # MDM cJSON requires floor + pow, in the C math lib, so we add "m"
    TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets ssl crypto pthread dl z c m)
|}

Latest revision as of 20:02, 29 November 2016

Issues:

  • need cross compile toolchain
    • you need to find or build an entire toolchain environment; ubuntu has some, others have some, Crosstool-NG will build one from scratch
    • for autotools, you typically need to define tools like ar, gcc, g++ in the build script
    • for cmake, you typically need a toolchain file to specify details
  • need to cross compile any dependencies (static link what you can)
  • need to match the libraries on target machine for any dynamic linking requirements
  • need to match the build to the target kernel (or you might get "kernel too old")
libz example
   echo Getting source code for libraries...
   if [ ! -d zlib ]; then
       wget http://zlib.net/zlib-${zlib_version}.tar.gz
       untar zlib-${zlib_version}.tar.gz
       ln -s zlib-${zlib_version} zlib
   fi
   echo Building libraries...
   if [ "$rebuild_libs" = true ]; then
   
       export cross=arm-linux-gnueabi-
   
       # ZLIB
       cd zlib
       ./configure --prefix=${install_root}
       make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib"
       make install
       cd ..
   fi
openssl example
   echo Getting source code for libraries...
   if [ ! -d openssl ]; then
       wget https://www.openssl.org/source/openssl-${openssl_version}.tar.gz
       untar openssl-${openssl_version}.tar.gz
       ln -s openssl-${openssl_version} openssl
   fi
   
   echo Building libraries...
   
   if [ "$rebuild_libs" = true ]; then
   
       export cross=arm-linux-gnueabi-
   
       # OPENSSL
       cd openssl
       ./Configure dist --prefix=${install_root} --openssldir=${install_root}
       make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib"
       make install
       cd ..
   fi
multi-lib complete example
   # Cross compile env for armv4 chipset 
   # Conel cellular modem (Verizon/Pogens)
   # ARM 926EJ-S processor
   # Using toolchain on vpn.causam.com that was built a while ago by Bryan G. with Crosstool-NG 1.20.0
   
   # We will cross compile these libraries:
   # zlib for compression
   # openssl for encryption - NOTE that it is considered unsafe to bake zlib directly into openssl
   # libwebsockets for wss
   
   
   # LIB VERSIONS: No reason I know of not to use the latest
   # It's just that the versions are needed in the script
   zlib_version=1.2.8     # see http://www.zlib.net/
   openssl_version=1.1.0c # see https://www.openssl.org/source/
   lws_branch=v2.0-stable
   
   
   # Turn this on at least at the first run, again as needed.
   rebuild_libs=true
   
   
   # ==========================================
   # cross compile toolchain magic happens here
   # ==========================================
   
   nickname=armv4
   
   # We will use a cross-compile toolchain that was built a while ago by Bryan G. with Crosstool-NG 1.20.0
   # We will build on an older machine (vpn.causam.com) since this did not seem to help limit the kernel: --enable-kernel=${max_kernel}
   if [ ! -d /opt/crosstool-source-v4 ]; then
       echo "Cross-compile toolchain not found..."
       kill -SIGINT $$;
   fi
   
   export cross=arm-926ejs-linux-gnueabi-
   CC=${cross}gcc
   CMAKE_TOOLCHAIN_FILE=/home/m/development/causam/git/np/nop-bigress-client-c/${nickname}/toolchain.cmake
   export PATH=/opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/bin:${PATH}
   
   export install_root=/home/m/development/${nickname}
   
   # ==========================================
   
   
   echo Getting source code for libraries...
   cd ${install_root}/source
   if [ ! -d zlib ]; then
       wget http://zlib.net/zlib-${zlib_version}.tar.gz
       tar -xzvf zlib-${zlib_version}.tar.gz
       ln -s zlib-${zlib_version} zlib
   fi
   if [ ! -d openssl ]; then
       wget https://www.openssl.org/source/openssl-${openssl_version}.tar.gz
       tar -xzvf openssl-${openssl_version}.tar.gz
       ln -s openssl-${openssl_version} openssl
   fi
   if [ ! -d libwebsockets ]; then
       git clone https://github.com/warmcat/libwebsockets.git
       cd libwebsockets
       git checkout ${lws_branch}
       cd ..
   fi
   
   echo Building libraries...
   
   if [ "$rebuild_libs" = true ]; then
   
       # Good help: http://stackoverflow.com/questions/11841919/cross-compile-openssh-for-arm
       # BUT full of landmines...    
       # Use [file ##.so.##] or [file executable] to find out the architecture of an asset.
   
       # ZLIB
       # THIS IS EXTREMELY BRITTLE, do not adjust
       # THIS DOES NOT USE 'ar r', IT USES 'ar', BUT OPENSSL WILL USE 'ar r'
       # Make sure we build static
       # NOTE the global CC define WILL NOT WORK HERE, use this CC define specifically or it WILL BREAK
       cd zlib
       CC="${cross}gcc" AR="${cross}ar" RANLIB="${cross}ranlib" ./configure --static --prefix=${install_root}
   
       make clean
       make
       make install
       cd ..
       
       # OPENSSL
       # THIS IS EXTREMELY BRITTLE, do not adjust
       cd openssl
       ./Configure dist --prefix=${install_root} --openssldir=${install_root}
       make clean
       make CC="${cross}gcc" AR="${cross}ar r" RANLIB="${cross}ranlib"
       make install
       cd ..
   
       # LIBWEBSOCKETS
       # NOTE: we basically need most lws features, with the exception of PLUGINS support.
       cd libwebsockets
       git pull
       mkdir -p cmake-${nickname}-release
       cd cmake-${nickname}-release
       rm -rf CMakeCache.txt CMakeFiles cmake_install.cmake *.cpb Makefile
       cmake -static \
           -DCMAKE_INSTALL_PREFIX:PATH=${install_root}         \
           -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}      \
           -DZLIB_LIBRARY=${install_root}/lib                  \
           -DZLIB_INCLUDE_DIR=${install_root}/include          \
           -DLWS_OPENSSL_LIBRARIES=${install_root}/lib         \
           -DLWS_OPENSSL_INCLUDE_DIRS=${install_root}/include  \
           -DLWS_WITH_SHARED=OFF           \
           -DLWS_WITHOUT_TESTAPPS=1        \
           -DLWS_WITH_SSL=1                \
           -DLWS_WITH_ZLIB=1               \
           -DLWS_WITHOUT_EXTENSIONS=0      \
           -DLWS_HAVE_SSL_CTX_set1_param=1 \
           -DLWS_WITHOUT_DAEMONIZE=0       \
           -DLWS_WITHOUT_SERVER=1          \
           -DLWS_MAX_SMP=4                 \
           ..
       make
       make install
       cd ../..
   fi
   
   echo =========================================================
   echo Cross-compiled libraries have been built here [${install_root}/lib]:
   echo =========================================================
   ls -la ${install_root}/lib
   echo =========================================================
   
   echo Copy nop-client source and toolchain CMakeLists.txt...
   mkdir -p nop-client
   cp -rp /home/m/development/causam/git/np/nop-bigress-client-c/* nop-client/
   cp -rp /home/m/development/causam/git/np/nop-bigress-client-c/${nickname}/CMakeLists.txt nop-client/
   cd nop-client
   mkdir -p cmake-${nickname}-release
   cd cmake-${nickname}-release
   
   echo Building nop-client...
   rm -rf CMakeCache.txt CMakeFiles cmake_install.cmake *.cpb Makefile
   cmake -static \
       -DCMAKE_INSTALL_PREFIX:PATH=${install_root}         \
       -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}      \
       ..
   make
   
   echo =========================================================
   echo Binary details of build...
   echo =========================================================
   file nop-client
   echo =========================================================
   
   echo Copying to remote device...
   ssh -p 2222 root@sp-c703.energynet.io "cat > nop-client-${nickname}" < nop-client
   ssh -p 2222 root@sp-c703.energynet.io "chmod a+x nop-client-${nickname}"
   
   # Here are some other remote commands
   #
   # rm the bin (had to do this when cp failed due to lack of space on device)
   # ssh -p 2222 root@sp-c703.energynet.io "rm nop-client-${nickname}"
   #
   # cp FROM the device back to host (again, to clean space on device)
   # ssh -p 2222 root@sp-c703.energynet.io "cat nop-client.0" > nop-client.0
cmake toolchain file example
   # https://github.com/zyga/cmake-toolchains/blob/master/Toolchain-Ubuntu-gnueabi.cmake
   # Sample toolchain file for building for ARM from an Ubuntu Linux system.
   #
   # Typical usage:
   #    *) install cross compiler: `sudo apt-get install gcc-arm-linux-gnueabi`
   #    *) cd build
   #    *) cmake -DCMAKE_TOOLCHAIN_FILE=~/Toolchain-Ubuntu-gnueabi.cmake ..
   
   set(CMAKE_SYSTEM_NAME Linux)
   set(TOOLCHAIN_PREFIX arm-926ejs-linux-gnueabi)
   
   # cross compilers to use for C and C++
   set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc)
   set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++)
   
   # target environment on the build host system
   #   set 1st to dir with the cross compiler's C/C++ headers/libs
   set(CMAKE_FIND_ROOT_PATH /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi)
   
   # modify default behavior of FIND_XXX() commands to
   # search for headers/libs in the target environment and
   # search for programs in the build host environment
   set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
   set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
   set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
   
CMakeLists.txt example
   cmake_minimum_required(VERSION 2.8)
   # MDM silently append relative paths to absolute paths without warnings
   cmake_policy(SET CMP0015 NEW)
   # MDM Set the executable name!
   # project(nop-bigress-client-c)
   project(nop-client)
   # -------------------------------
   # To ensure we get a static build.
   # For this to work, the linker needs to be able to find all libraries.
   # Make sure they are included next in [include|link]_directories.
   # Also make sure they are included in TARGET_LINK_LIBRARIES.
   # -------------------------------
   SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
   SET(BUILD_SHARED_LIBRARIES OFF)
   SET(CMAKE_EXE_LINKER_FLAGS "-static")
   # -------------------------------
   # -------------------------------
   # MDM For cross compile, we must provide all lib paths.
   # We have ours: /home/m/development/armv4
   # And the toolchain is here: /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi
   # -------------------------------
   include_directories(/home/m/development/armv4/include /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/include)
   link_directories(/home/m/development/armv4/lib /opt/crosstool-source-v4/arm-926ejs-linux-gnueabi/lib)
   # -------------------------------
   # -------------------------------
   # MDM Build *everything* in ./src
   # -------------------------------
   # WARNING: [make] won't pick up new files until you rerun [cmake].
   # Change this to be explicit if that becomes a problem.
   # Easy to change:
   #    set( NOP_SOURCES src/nop_client.c src/helpers.c )
   aux_source_directory(./src NOP_SOURCES)
   add_executable(
       ${PROJECT_NAME}
       ${NOP_SOURCES}
   )
   # The main linker line used to create the executable
   # MDM added [c] after ubuntu xenial started giving segfaults in dl -> openssl - didn't help!
   # see http://stackoverflow.com/questions/7526255/unknown-reference-to-dlopen-in-dlopen
   # TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets ssl crypto pthread dl z c)
   # MDM NO LINKING since we built websockets static!  no, this doesn't work...
   #TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets)
   # MDM cJSON requires floor + pow, in the C math lib, so we add "m"
   TARGET_LINK_LIBRARIES(${PROJECT_NAME} websockets ssl crypto pthread dl z c m)