参考文档
- http://mxnet.incubator.apache.org/install/index.html mxnet官方install手册
本文档涉及到的目标硬件为全志H8(8核Cortex-A7，Armv7架构)，但是对其他Arm芯片也有一定的借鉴意义，只需要更换交叉编译链即可。
开发环境介绍
- 主机操作系统：Ubuntu14.04 64位
- 目标平台：CQA83t 全志H8
- 交叉工具链：arm-linux-gnueabihf，gcc4.9.2
- mxnet版本：20180124-master
注意：mxnet的最低GCC版本为4.8.0
设置交叉编译链
# 在/etc/bash.bashrc的最后增加如下指令
# Allwinner H8 CQA83t cross compiler
export ARCH=arm
export PATH=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/:$PATH
export CROSS_COMPILE=arm-linux-gnueabihf-
export CC=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-gcc
export CXX=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-g++
export LD=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-ld
export AR=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-ar
export AS=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-as
export RANLIB=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-ranlib
# 修改完成之后需要重启命令行才能生效
# 你可以通过如下指令来确认交叉编译链是否已经设置好
echo $CC
# 当显示/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-gcc时表示交叉编译链
# 已经设置好
# 当需要更换为本机编译时屏蔽上面的指令即可
下载依赖库
| 依赖库 | 版本 | 下载地址 |
|---|---|---|
| openblas | 0.2.20 | https://github.com/xianyi/OpenBLAS |
编译OpenBLAS
编译
cd OpenBLAS-0.2.20
# 编译单独的静态库版本
make TARGET=ARMV7 HOSTCC=gcc CROSS=1 CROSS_SUFFIX=arm-linux-gnueabihf- BINARY=32 NOFORTRAN=1 libs -j8
# 编译动态库版本
make TARGET=ARMV7 HOSTCC=gcc CROSS=1 CROSS_SUFFIX=arm-linux-gnueabihf- BINARY=32 NOFORTRAN=1 libs shared
# 另附上ARMV8芯片的编译命令以供参考
make TARGET=ARMV8 HOSTCC=gcc CROSS=1 CROSS_SUFFIX=aarch64-linux-gnu- BINARY=64 NOFORTRAN=1 libs -j8
make TARGET=ARMV8 HOSTCC=gcc CROSS=1 CROSS_SUFFIX=aarch64-linux-gnu- BINARY=64 NOFORTRAN=1 libs shared -j8
安装
# PREFIX使用您自己的自定义安装目录
make PREFIX=/usr/local/H8/openBLAS-0.2.20 install
编译mxnet
修改CmakeLists.txt
修改
# ---[ LAPack
if(USE_LAPACK)
add_definitions(-DMXNET_USE_LAPACK=1)
list(APPEND mxnet_LINKER_LIBS lapack)
else(USE_LAPACK)
Workaround for Windows until using new Jenkinsfile.
if(BLAS STREQUAL "Open" OR BLAS STREQUAL "open")
add_definitions(-DMXNET_USE_LAPACK=1)
endif()
endif()
为
# ---[ LAPack
if(USE_LAPACK)
add_definitions(-DMXNET_USE_LAPACK=1)
list(APPEND mxnet_LINKER_LIBS lapack)
else(USE_LAPACK)
# Workaround for Windows until using new Jenkinsfile.
# if(BLAS STREQUAL "Open" OR BLAS STREQUAL "open")
# add_definitions(-DMXNET_USE_LAPACK=1)
# endif()
endif()
# 这里屏蔽add_definitions(-DMXNET_USE_LAPACK=1)是因为这里的添加是Windows专用的,针对Linux需要屏蔽
修改tests/CmakeLists.txt
修改
if(NOT MSVC)
set(UNITTEST_STATIC_LINK ON)
endif()
为
if(NOT MSVC)
set(UNITTEST_STATIC_LINK OFF)
endif()
修改
if(UNITTEST_STATIC_LINK)
target_link_libraries(${PROJECT_NAME}_unit_tests
${GTEST_LIBRARY}
${BEGIN_WHOLE_ARCHIVE} mxnet ${END_WHOLE_ARCHIVE}
dmlc
${mxnet_LINKER_LIBS}
${pslite_LINKER_LIBS}
)
else()
target_link_libraries(${PROJECT_NAME}_unit_tests
${GTEST_LIBRARY}
dmlc
${nnvm_LINKER_LIBS}
${mxnet_LINKER_LIBS}
mxnet
${pslite_LINKER_LIBS}
)
endif()
为
if(UNITTEST_STATIC_LINK)
target_link_libraries(${PROJECT_NAME}_unit_tests
${GTEST_LIBRARY}
${BEGIN_WHOLE_ARCHIVE} mxnet ${END_WHOLE_ARCHIVE}
dmlc
${mxnet_LINKER_LIBS}
${pslite_LINKER_LIBS}
)
else()
target_link_libraries(${PROJECT_NAME}_unit_tests
${GTEST_LIBRARY}
dmlc
${nnvm_LINKER_LIBS}
${mxnet_LINKER_LIBS}
mxnet
${pslite_LINKER_LIBS}
rt
)
endif()
新建cmake Build目录
cd incubator-mxnet
mkdir _install
打开cmake-gui进行交叉编译链配置
注意：mxnet要求cmake版本需要高于3.0.2
选择Specify options for cross-compiling，进入交叉编译链设置界面
# 设置交叉编译相关参数
# 本文使用的是NVIDIA提供的交叉工具链，存放位置为/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/
# (更改为你正在使用的目录)
# 设置Operating System 为 arm-linux
# 设置C编译器为 /opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-gcc
# (更改为你正在使用的gcc)
# 设置C++编译器为 /opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-g++
# 设置System Root 为 /opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/arm-linux-gnueabihf/sysroot
点击Configure生成配置选项，并且打开Advancd选项
此时Cmake会显示CUDA错误，暂时先不用管，后面修改设置后错误会消失
修改设置选项：
# 去掉勾选为FLASE 勾选为TRUE
USE_CUDA # FALSE
USE_CUDNN # FALSE
USE_GPERFTOOLS # FALSE
USE_JEMALLOC # FALSE
USE_LAPACK # FALSE
USE_OPENCV # FALSE
USE_OPENMP # TRUE 多核ARM建议开启openmp
点击Configure
再次修改设置选项：
# 指定安装类型和目录
CMAKE_BUILD_TYPE Release
CMAKE_INSTALL_PREFIX /usr/local/H8/mxnet-20180122
CMAKE_C_FLAGS -mfpu=neon -march=armv7-a
CMAKE_CXX_FLAGS -mfpu=neon -march=armv7-a
BLAS Open
# 指定OpenBLAS头文件和库文件路径(填写之前交叉编译OpenBLAS的安装目录)
OpenBLAS_INCLUDE_DIR /usr/local/H8/openBLAS-0.2.20/include
OpenBLAS_LIBRARY /usr/local/H8/openBLAS-0.2.20/lib/libopenblas.so
点击Configure
点击Generate生成Makefile
进入_install目录&编译
cd _install
make -j8
make install
查看libmxnet.so的依赖库
arm-linux-gnueabihf-readelf -d /usr/local/H8/mxnet-20180122/lib/libmxnet.so
Dynamic section at offset 0x13fde68 contains 33 entries:
Tag Type Name/Value
0x00000001 (NEEDED) Shared library: [libopenblas.so.0]
0x00000001 (NEEDED) Shared library: [3rdparty/openmp/runtime/src/libomp.so]
0x00000001 (NEEDED) Shared library: [libpthread.so.0]
0x00000001 (NEEDED) Shared library: [libdl.so.2]
0x00000001 (NEEDED) Shared library: [libstdc++.so.6]
0x00000001 (NEEDED) Shared library: [libm.so.6]
0x00000001 (NEEDED) Shared library: [libgomp.so.1]
0x00000001 (NEEDED) Shared library: [libgcc_s.so.1]
0x00000001 (NEEDED) Shared library: [libc.so.6]
0x00000001 (NEEDED) Shared library: [ld-linux-armhf.so.3]
从上面的依赖库可以看到3rdparty/openmp/runtime/src/libomp.so这个相对路径，这样并不利于最终的部署测试，在部署时会出现无法找到libomp.so的问题
可以尝试改动Cmake生成的Makefile来解决
修改_install/CMakeFiles/mxnet.dir/link.txt
将3rdparty/openmp/runtime/src/libomp.so
修改为-L/usr/local/H8/mxnet-20180122/lib/ -lomp
# /usr/local/H8/mxnet-20180122/lib/ 即是之前的安装目录
重新进行一次编译&安装
make -j8
make install
开启NNPACK
如何开启mxnet的nnpack编译，官方的编译教程只提供了直接Make的方案，这里介绍一下使用Cmake来开启nnpack可选组件编译的方法
修改设置选项：
CMAKE_CXX_FLAGS_RELEASE -O3 -DNDEBUG -DMXNET_USE_NNPACK=1 -I/usr/local/S912/Ubuntu/nnpack-20180111/include
CMAKE_C_FLAGS_RELEASE -O3 -DNDEBUG -DMXNET_USE_NNPACK=1 -I/usr/local/S912/Ubuntu/nnpack-20180111/include
点击Generate重新进行一次编译
/usr/local/S912/Ubuntu/nnpack-20180111/lib/libnnpack.a /usr/local/S912/Ubuntu/nnpack-20180111/lib/libpthreadpool.a /usr/local/S912/Ubuntu/nnpack-20180111/lib/libcpuinfo.a