在x86虚拟机搭建arm64交叉编译环境记录

在x86虚拟机搭建arm64交叉编译环境记录

依赖安装

sudo apt-get install gcc-5-aarch64-linux-gnu g++-5-aarch64-linux-gnu gdb-multiarch qemu-user-static -y
1

基础文件系统

以18.04为例（注意：Nvidia相关板子在其官网有文件系统可供下载）

# 下载基础文件系统
wget –no-check-certificate https://mirrors.tuna.tsinghua.edu.cn/ubuntu-cdimage/ubuntu-base/releases/18.04.5/release/ubuntu-base-18.04.5-base-arm64.tar.gz

# 创建交叉编译文件系统目录
mkdir XRoot
# 将文件系统解压到创建的目录
sudo tar –xzf ubuntu-base-18.04.5-base-arm64.tar.gz –C XRoot
# 复制模拟器
sudo cp /usr/bin/qemu-aarch64-static XRoot/usr/bin
# 切换根目录（进入交叉编译文件系统），可以像使用实体机一样使用该文件系统，在其中安装ros等依赖。
sudo chroot XRoot
1
2
3
4
5
6
7
8
9
10
11

toolchain.cmake

一些变量的定义解释见文章：cmake交叉编译配置文件 - XXX.toolchain.cmake

示例：

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_C_COMPILER /usr/bin/aarch64-linux-gnu-gcc)
set(CMAKE_CXX_COMPILER /usr/bin/aarch64-linux-gnu-g++)
# 一般设置为工具链的sysroot目录
set(CMAKE_SYSROOT ${CMAKE_CURRENT_LIST_DIR})
# 指定了一个或者多个优先于其他搜索路径的搜索路径
# 指定多个：set(CMAKE_FIND_ROOT_PATH /opt/arm /opt/inst)
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})
# 有三种取值：
# NEVER表示不在你CMAKE_FIND_ROOT_PATH下进行查找
# ONLY表示只在这个路径下查找
# BOTH表示先查找这个路径，再查找全局路径
# 对于这个变量来说，一般都是调用宿主机的程序，所以一般都设置成NEVER
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
# 一般来说也是ONLY,如果你想改变，一般也是在相关的FIND命令中增加option来改变局部设置,有NO_CMAKE_FIND_ROOT_PATH,ONLY_CMAKE_FIND_ROOT_PATH,BOTH_CMAKE_FIND_ROOT_PATH
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
# 这里需要设置成ONLY来保证我们的库是在交叉环境中找的
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

set(CMAKE_PREFIX_PATH ${CMAKE_FIND_ROOT_PATH} ${CMAKE_SYSROOT}/opt/ros/melodic)
set(CMAKE_LIBRARY_PATH ${CMAKE_SYSROOT}/usr/local/lib ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/blas ${CMAKE_SYSROOT}/usr/lib)
set(CMAKE_INCLUDE_PATH ${CMAKE_SYSROOT}/usr/local/include ${CMAKE_SYSROOT}/usr/include/aarch64-linux-gnu ${CMAKE_SYSROOT}/usr/include)
set(TARGET_ARCHITECTURE "arm64")

# 配置Qt和Boost
SET(QT_QMAKE_EXECUTABLE /opt/qt-embedded/qmake)
SET(BOOST_ROOT /opt/boost_arm)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

搜索PATHS选项中定义的路径

• CMake变量CMAKE_FIND_ROOT_PATH指定一个或多个目录作为所有其他搜索目录的前缀。该变量re-root了整个搜索过程到指定位置。默认情况下，CMAKE_FIND_ROOT_PATH为空。

• CMAKE_SYSROOT变量也可以用来指定一个目录作为搜索目录前缀

• 上述两个变量在交叉编译时，用于指定搜索根路径很有用。

  • 默认先搜索CMAKE_FIND_ROOT_PATH，之后搜索CMAKE_SYSROOT，最后搜索其他non-rooted路径
  • 通过配置CMAKE_FIND_ROOT_PATH_MODE_INCLUDE变量修改上述搜索行为：
  • CMAKE_FIND_ROOT_PATH_BOTH：按上述顺序搜索
  • NO_CMAKE_FIND_ROOT_PATH：不搜索CMAKE_FIND_ROOT_PATH指定的路径
  • ONLY_CMAKE_FIND_ROOT_PATH：只搜索re-rooted目录和CMAKE_STAGING_PREFIX路径下的目录

默认的搜索顺序设计时考虑了最常用到最不常用的场景，实际工程中可以通过调用多次带有NO_*选项的命令来修改搜索顺序。一旦搜索成功，则结果将被缓存下来。

# 具体用法可以通过 sourcegraph 搜索
find_library(PYTORCH_LIBRARY pytorch_jni
  PATHS ${PYTORCH_LINK_DIRS}
  NO_CMAKE_FIND_ROOT_PATH)
1
2
3
4

问题

1.下载文件系统，拷贝了qemu，在chroot之后执行apt update报错

# apt-get update
Get:1 http://security.debian.org stretch/updates InRelease [94.3 kB]                                                                         
0% [1 InRelease gpgv 94.3 kB] [Waiting for headers] [Waiting for headers] [Waiting for headers]gpgv: Signature made Tue Apr  9 18:21:53 2019 UTC
gpgv:                using RSA key D21169141CECD440F2EB8DDA9D6D8F6BC857C906
gpgv: Good signature from "Debian Security Archive Automatic Signing Key (8/jessie) "
gpgv: Signature made Tue Apr  9 18:21:53 2019 UTC
gpgv:                using RSA key A1BD8E9D78F7FE5C3E65D8AF8B48AD6246925553
gpgv: Good signature from "Debian Archive Automatic Signing Key (7.0/wheezy) "
/usr/bin/apt-key: 596: /usr/bin/apt-key: cannot create /dev/null: Permission denied
terminate called after throwing an instance of 'std::runtime_error'
  what():  random_device::random_device(const std::string&)
qemu: uncaught target signal 6 (Aborted) - core dumped
Aborted (core dumped)
1
2
3
4
5
6
7
8
9
10
11
12
13

解决方案：https://elinux.org/Jetson/TX1_Sample_Root_Filesystem，按照文章中步骤操作，直至执行上述命令不报错

2.chroot后切换国内源报错

Err:5 https://mirrors.ustc.edu.cn/ubuntu-ports bionic Release
  Certificate verification failed: The certificate is NOT trusted. The certificate issuer is unknown.  Could not handshake: Error in the certificate verification. [IP: 202.141.160.110 443]
1
2

解决方案：

touch etc/apt/apt.conf.d/99verify-peer.conf \
&& echo >>etc/apt/apt.conf.d/99verify-peer.conf "Acquire { https::Verify-Peer false }"
1
2

3.chroot警告

bash: warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8)
1

解决方案：

sudo LC_ALL=C chroot . /bin/bash
1

4.交叉编译环境出现.so链接文件找不到问题

报错示例：

CMake Error at /home/colin/nx-cs/opt/ros/melodic/share/catkin/cmake/assert.cmake:20 (message):

  Assertion failed: file
  '/home/colin/nx-cs/usr/lib/aarch64-linux-gnu/librt.so' does not exist.
  Message: RT Library

Call Stack (most recent call first):
  /home/colin/nx-cs/opt/ros/melodic/share/catkin/cmake/tools/rt.cmake:44 (assert_file_exists)
  /home/colin/nx-cs/opt/ros/melodic/share/catkin/cmake/all.cmake:164 (include)
  /home/colin/nx-cs/opt/ros/melodic/share/catkin/cmake/catkinConfig.cmake:20 (include)
  CMakeLists.txt:58 (find_package)
1
2
3
4
5
6
7
8
9
10
11

make[2]: *** No rule to make target '/usr/lib/aarch64-linux-gnu/libz.so', needed by '/home/colin/ros_ws/catkin_ws/devel/lib/ud_survey_comm/ud_survey_
CMakeFiles/Makefile2:4114: recipe for target 'ud_survey_comm/CMakeFiles/ud_survey_comm_node.dir/all' failed
1
2

解决方案一

解决方案二：

将编译过程中报错的动态库，在宿主机相关目录创建软链接，如：

cd /usr/lib/aarch64-linux-gnu
sudo ln -s  ~/nx-cs/lib/aarch64-linux-gnu/libpcre.so.3 libpcre.so
sudo ln -s  ~/nx-cs/lib/aarch64-linux-gnu/libz.so.1.2.11 libz.so
1
2
3

5.一定会遇到的问题（ROS中cmake文件和pc文件的绝对路径问题）

报错示例：

/opt/ros/melodic/lib/libtf.so: error adding symbols: File in wrong format
1

描述：ros中的库的.cmake等文件都是用了绝对路径，需要批量改为相对路径

解决方案：

# 宿主机内执行
# s#/usr/include/#${CMAKE_SYSROOT}/usr/include/#g 的意思是将"/usr/include/"字段替换为"${CMAKE_SYSROOT}/usr/include/"
# ~/Documents/CrossSysroot 需要改为自己实际目录

sudo sed -i 's#/usr/include/#${CMAKE_SYSROOT}/usr/include/#g' `find ~/Documents/CrossSysroot/opt/ros/melodic -type f -name "*.cmake"`

# chroot 后执行， 此时sed命令不需要加sudo权限，注意toolchain.cmake文件可能同时被修改
sed -i 's/opt/ros/melodic#/home/colin/nx-cs/opt/ros/melodic#g' `find ./ -type f -name "*.cmake"`

sudo sed -i 's#prefix=/opt/ros/melodic#prefix=/home/ubuntu/Documents/CrossSysroot/opt/ros/melodic#g' `find ./ -type f -name "*.pc"`

sudo sed -i 's#prefix=/usr#prefix=/home/ubuntu/Documents/CrossSysroot/usr#g' `find ./ -type f -name "*.pc"`
1
2
3
4
5
6
7
8
9
10
11
12