centos &promxox 使用qemu-system-aarch64 安装麒麟V10 ARM版本

centos& proxmox 使用qemu-system-aarch64 安装麒麟V10 ARM版本

关于为什么要使用proxmox 模拟ARM 虚机而不直接使用centos

大致分为两个方面

1、虚机管理上面，proxmox是专业的虚拟化平台，可以直接链接克隆，等高级虚机管理功能，使用更加方便

2、效能方面，同样底层是qemu,但是使用proxmox模拟ARM，proxmox 比centos 高出很多

更多想了解proxmox可以百度去了解一下

，

1、安装pytone3.10

1.1、在python官网下载所需的python3

mkdir -p /home/python3
cd  /home/python3
wget --no-check-certificate https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tar.xz

1.2、下载python3依赖安装包（提前下载，拷贝到离线环境的内网机器中）

yum install yum-plugin-downloadonly
yum reinstall --downloadonly --downloaddir=/home/python3 zlib-devel bzip2-devel openssl-devel ncurses-devel  epel-release gcc gcc-c++ xz-devel readline-devel gdbm-devel sqlite-devel tk-devel db4-devel libpcap-devel libffi-devel

1.3 、离线安装python3及依赖安装包

cd /home/python3
rpm -Uvh ./*.rpm --nodeps --force

1.4、编译安装python3安装包

cp /home/Python-3.10.2.tar.xz /home/python3/
tar -xvf Python-3.10.2.tar.xz
mkdir -p /usr/local/python3
./configure --prefix=/usr/local/python3
cd Python-3.10.2/
./configure --prefix=/usr/local/python3
make && make install

1.5、配置python3系统路径

ln -s /usr/local/python3/bin/python3 /usr/local/bin/python3
ln -s /usr/local/python3/bin/pip3 /usr/local/bin/pip3
echo 'PATH=$PATH:/usr/local/bin' >> /etc/profile   或者 echo "PATH=\$PATH:/usr/local/bin" >> /etc/profile
source /etc/profile

1.6 、验证是否安装成功

[root@slave3 ~]# python3 -V
Python 3.10.2
[root@slave3 ~]# pip3 -V
pip 21.2.4 from /usr/local/python3/lib/python3.10/site-packages/pip (python 3.10)

2、安装qemu7.1

2.1、yum安装交叉编译工具

yum install -y binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu bison flex glib2 glib2-devel pixman-devel bc  bzip2
wget http://springdale.princeton.edu/data/springdale/7/x86_64/os/Computational/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm
rpm -Uvh glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm

2.2、安装re2c

cd /home/qemu7.1/
wget https://down.24kplus.com/linux/re2c-1.1.1.tar.gz
tar -zxvf re2c-1.1.1.tar.gz
cd re2c-1.1.1
./configure && make && make install
re2c -v

2.3、安装ninja

cd /home/qemu7.1/
wget https://codeload.github.com/ninja-build/ninja/tar.gz/refs/tags/v1.10.2
tar -zxvf v1.10.2
cd ninja-1.10.2/
./configure.py --bootstrap
cp /home/qemu7.1/ninja-1.10.2/ninja /usr/bin
ninja --verison

2.4、升级gcc g++

yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/centos-release-scl-rh-2-3.el7.centos.noarch.rpm
yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/centos-release-scl-2-3.el7.centos.noarch.rpm
yum install -y devtoolset-9-gcc-c++
scl enable devtoolset-9 bash

2.5、安装qemu7.1.0

wget https://download.qemu.org/qemu-7.1.0.tar.xz
cp qemu-7.1.0.tar.xz /home/qemu7.1/
cd /home/qemu7.1/
tar -xvf qemu-7.1.0.tar.xz
cd qemu-7.1.0/
./configure --enable-kvm --enable-debug   --enable-strip --target-list=aarch64-softmmu
make -j4
make install

2.6、检查qemu安装

[root@slave3 ~]# which qemu-system-aarch64 
/usr/local/bin/qemu-system-aarch64

3、安装麒麟V10 ARM版本

3.1 下载对应的版本

[root@base ~]# cd /mnt/nfs/template/iso/
[root@base ~]# wget https://distro-images.kylinos.cn:8802/web_pungi/download/share/BP9pZlFhKjANkwoWrsgETDMXLmOxait1/Kylin-Server-10-SP2-Release-Build09-20210524-arm64.iso
[root@base ~]# ssh root@slave3 
[root@slave3 ~]# mkdir -p /mnt/nfs
[root@slave3 ~]# mount.nfs 172.16.104.111:/nfs/pve /mnt/nfs/
[root@slave3 ~]# ll -h /mnt/nfs/template/iso/
-rw-r--r-- 1 root root  54K 4月  16 2014 axel-2.4-9.el7.x86_64.rpm
-rw-r--r-- 1 root root 4.5G 9月  21 16:06 CentOS-7-x86_64-DVD-2003.iso
-rw-r--r-- 1 root root 4.4G 9月  21 18:03 CentOS-7-x86_64-DVD-2009.iso
-rw-r--r-- 1 root root  52K 9月  23 14:42 ks.iso  #尝试pex 和有盘的无人值守安装均在暗中过程中报错
-rw-r--r-- 1 root root 4.0G 3月  25 2022 Kylin-Server-10-SP2-Release-Build09-20210524-arm64.iso
-rw-r--r-- 1 root root 995M 9月  20 11:47 proxmox-ve_7.2-1.iso
-rw-r--r-- 1 root root 1.4G 9月  21 14:37 ubuntu-22.04.1-live-server-arm64.iso

3.2 、下载qemu_efi.fd，创建磁盘

[root@slave3 ]#mkdir /root/kylinv10
[root@slave3 ]#cd /root/kylinv10/
[root@slave3 kylinv10]wget https://releases.linaro.org/components/kernel/uefi-linaro/latest/release/qemu64/QEMU_EFI.fd --no-check-certificate
[root@slave3 kylinv10]# df -TH
文件系统                类型      容量  已用  可用 已用% 挂载点
devtmpfs                devtmpfs   14G     0   14G    0% /dev
tmpfs                   tmpfs      14G     0   14G    0% /dev/shm
tmpfs                   tmpfs      14G  287M   14G    3% /run
tmpfs                   tmpfs      14G     0   14G    0% /sys/fs/cgroup
/dev/mapper/centos-root xfs        97G   15G   83G   16% /
/dev/sda1               xfs       1.1G  237M  827M   23% /boot
/dev/mapper/centos-home xfs        29G  2.6G   27G    9% /home
[root@slave3 kylinv10]qemu-img create -f qcow2 kylin_arm64.qcow2 60G

3.3 、启动虚拟机

3.3.1 、参数1 QEMU使用VNC远程图形界面进行安装

注意 -vnc 0.0.0.0:0 监控端口号为0.0.0.0:5900 如果不加监听端口号为127.0.0.1:5900 出本机外其他机器不能连接

我第一次没加-vnc 0.0.0.0:0 参数，使用rinetd 进行端口转发 rinetd j具体的用法可以百度。

qemu-system-aarch64 -m 2048 -cpu cortex-a76 -smp 4,sockets=1,cores=4 -M virt  \
-bios /root/kylinv10/QEMU_EFI.fd  -device VGA -vnc 0.0.0.0:0  -device nec-usb-xhci -device usb-mouse -device usb-kbd \
-drive if=none,file=/root/kylinv10/kylin_arm64.qcow2,id=hd0 \
-device virtio-blk-device,drive=hd0  \
-drive if=none,file=/mnt/nfs/template/iso/KylinV10.iso,id=cdrom,media=cdrom \
-device virtio-scsi-device \
-device scsi-cd,drive=cdrom \
-net nic -net user,hostfwd=tcp::2222-:22\

3.3.1.1 图形界面后续安装和centos一样

从这个界面到下一个界面大概30分钟耐心等待

3.3.3.2 安装配置

3.3.3.3 安装配置

3.3.3.4安装过程在这里卡了半个小时，估计就是慢

3.3.3.5安装完成

3.3.3.6重启点击许可证，再次重启进入桌面

3.3.2、参考2 QEMU使用文本界面进行安装

由于麒麟V10的参数还没有研究，使用文本模式安装到分区的时候一直报错，不成功。

qemu-system-aarch64 -m 2048 -cpu cortex-a76 -smp 4,sockets=1,cores=4 -M virt  \
-bios /root/kylinv10/QEMU_EFI.fd  -nographic  \
-drive if=none,file=/root/kylinv10/kylin_arm64.qcow2,id=hd0 \
-device virtio-blk-device,drive=hd0  \
-drive if=none,file=/mnt/nfs/template/iso/KylinV10.iso,id=cdrom,media=cdrom \
-device virtio-scsi-device \
-device scsi-cd,drive=cdrom \
-net nic -net user,hostfwd=tcp::2222-:22

3.3.3 启动麒麟文本安装方式1

1、安装的时候需要选择第一项按E键进入编辑模式

  Install Kylin Linux Advanced Server V10                                   
  Test this media & install Kylin Linux Advanced Server V10                
  Troubleshooting -->                                                      
                                                                           
                                                                                                                          

  Use the ^ and v keys to change the selection.                       
  Press 'e' to edit the selected item, or 'c' for a command prompt.

2、修改console=tty0 video=efifb:off video=VGA-1:640x480-32@60me 为console=ttyAMA0

setparams 'Install Kylin Linux Advanced Server V10'
    linux /images/pxeboot/vmlinuz inst.stage2=hd:LABEL=Kylin-Server-10 ro \
console=tty0 video=efifb:off video=VGA-1:640x480-32@60me
initrd /images/pxeboot/initrd.img


  Press Ctrl-x to start, Ctrl-c for a command prompt or Escape to     
  discard edits and return to the menu. Pressing Tab lists possible   
  completions.

3、修改后如下然后 Ctrl-x 启动麒麟安装程序耐心等待

setparams 'Install Kylin Linux Advanced Server V10'
    linux /images/pxeboot/vmlinuz inst.stage2=hd:LABEL=Kylin-Server-10 ro \
console=ttyAMA0
initrd /images/pxeboot/initrd.img


  Press Ctrl-x to start, Ctrl-c for a command prompt or Escape to     
  discard edits and return to the menu. Pressing Tab lists possible   
  completions.

3.3.4 启动麒麟文本安装方式2

选择Troubleshooting --> Install Kylin Linux Advanced Server V10 in basic graphics mode进入基本图形（文本）安装界面

  Install Kylin Linux Advanced Server V10                                   
  Test this media & install Kylin Linux Advanced Server V10                
  Troubleshooting -->                                                      
                                                                           
                                                                                                                          

  Use the ^ and v keys to change the selection.                       
  Press 'e' to edit the selected item, or 'c' for a command prompt.

Install Kylin Linux Advanced Server V10 in basic graphics mode
Rescue a Kylin Linux Advanced Server system



  Use the ^ and v keys to change the selection.                       
  Press 'e' to edit the selected item, or 'c' for a command prompt.   
  Press Escape to return to the previous menu.

3.3.5、麒麟安装配置

从启动到进入安装配置界面预计15到30分钟耐心等待

3.3.5.1、这个界面输入r 回车刷新

3.3.5.2、选择2文本模式

4、proxmox 通过导入麒麟ARM系统盘安装

4.1 创建虚拟机

4.2 、修改CPU类型并创建EFI磁盘,导入安装好的麒麟V10盘

root@slave3:~# qm set 105 -arch aarch64
update VM 105: -arch aarch64
root@slave3:~# qm   set 105 -efidisk0 local-lvm:0
update VM 105: -efidisk0 local-lvm:0
Logical volume "vm-105-disk-0" created.
transferred 0.0 B of 64.0 MiB (0.00%)
transferred 2.0 MiB of 64.0 MiB (3.12%)
transferred 4.0 MiB of 64.0 MiB (6.25%)
transferred 6.0 MiB of 64.0 MiB (9.38%)
transferred 8.0 MiB of 64.0 MiB (12.50%)
transferred 10.0 MiB of 64.0 MiB (15.62%)
transferred 12.0 MiB of 64.0 MiB (18.75%)
transferred 14.0 MiB of 64.0 MiB (21.88%)
transferred 16.0 MiB of 64.0 MiB (25.00%)
transferred 18.0 MiB of 64.0 MiB (28.12%)
transferred 20.0 MiB of 64.0 MiB (31.25%)
transferred 22.0 MiB of 64.0 MiB (34.38%)
transferred 24.0 MiB of 64.0 MiB (37.50%)
transferred 26.0 MiB of 64.0 MiB (40.62%)
transferred 28.0 MiB of 64.0 MiB (43.75%)
transferred 30.0 MiB of 64.0 MiB (46.88%)
transferred 32.0 MiB of 64.0 MiB (50.00%)
transferred 34.0 MiB of 64.0 MiB (53.12%)
transferred 36.0 MiB of 64.0 MiB (56.25%)
transferred 38.0 MiB of 64.0 MiB (59.38%)
transferred 40.0 MiB of 64.0 MiB (62.50%)
transferred 42.0 MiB of 64.0 MiB (65.62%)
transferred 44.0 MiB of 64.0 MiB (68.75%)
transferred 46.0 MiB of 64.0 MiB (71.88%)
transferred 48.0 MiB of 64.0 MiB (75.00%)
transferred 50.0 MiB of 64.0 MiB (78.12%)
transferred 52.0 MiB of 64.0 MiB (81.25%)
transferred 54.0 MiB of 64.0 MiB (84.38%)
transferred 56.0 MiB of 64.0 MiB (87.50%)
transferred 58.0 MiB of 64.0 MiB (90.62%)
transferred 60.0 MiB of 64.0 MiB (93.75%)
transferred 62.0 MiB of 64.0 MiB (96.88%)
transferred 64.0 MiB of 64.0 MiB (100.00%)
transferred 64.0 MiB of 64.0 MiB (100.00%)
efidisk0: successfully created disk 'local-lvm:vm-105-disk-0,size=64M'
scp root@base:/root/kylinv10/kylin_arm64.qcow2 /var/lib/vz/images/
qm importdisk 104 /var/lib/vz/images/kylin_arm64.qcow2 local-lvm --format=qcow2

4.3 修改配置文件

arch: aarch64
bios: ovmf
boot: order=scsi0;net0;scsi2
cores: 8
efidisk0: local-lvm:vm-105-disk-0,size=64M
memory: 8196
meta: creation-qemu=6.2.0,ctime=1664187142
name: kylin001
net0: virtio=CA:76:0E:59:4A:B9,bridge=vmbr0
numa: 0
ostype: l26
scsi0: local-lvm:vm-105-disk-1,backup=0,size=60G
scsi2: bash:iso/Kylin-Server-10-SP2-Release-Build09-20210524-arm64.iso,media=cdrom,size=4100816K
scsihw: virtio-scsi-pci
serial0: socket    ##添加
smbios1: uuid=104ec11d-7100-4feb-b953-f058f7e7f622
sockets: 1    ##添加
vga: serial0   ##添加
vmgenid: 80eb3ab7-5964-4d7a-8cd8-6cd5a8ce4363 ##删除

4.4 启动虚拟机并进入控制台

4.5 启动虚拟机并进入控制台

4.6 修改hostname，ip地址

[root@kylin143 ~]# nmcli conn show
NAME     UUID                                  TYPE      DEVICE
enp0s1   f9976a4a-da8d-83a7-a86e-493b320147b2  ethernet  -------   #原网卡，需要删除
enp0s18  f9976a4a-da8d-83a7-a86e-493b320147b3  ethernet  enp0s18   #查看网卡名称是否正确，磁盘导入后这里会有两个网卡
[root@kylin143 ~]#  mv /etc/sysconfig/network-scripts/ifcfg-enp0s1   /etc/sysconfig/network-scripts/ifcfg-enp0s18
[root@kylin143 ~]# vim /etc/sysconfig/network-scripts/ifcfg-enp0s18
TYPE=Ethernet
DEFROUTE=yes
IPV4_FAILURE_FATAL=yes
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=enp0s18   #enp0s1 改为enp0s18
DEVICE=enp0s18   #enp0s1 改为enp0s18
UUID=f9976a4a-da8d-83a7-a86e-493b320147b2   #删除这行
ONBOOT=yes
IPADDR=172.16.104.143
GATEWAY=172.16.104.254
PROXY_METHOD=none
BROWSER_ONLY=no
PREFIX=24
BOOTPROTO=static  #dhcp  改为static

5、proxmox 修改参数安装麒麟V10

5.1 参考4.1 除了不擅长200G的硬盘，其他一样

5.2 、和4.2完全一样

5.3、和4.3完全一样

5.4 修改启动参数

ps -ef |grep 105 获取启动参数然后粘贴到记事本改成如下格式

/usr/bin/qemu-system-aarch64 -id 105 -name kylin001 -no-shutdown  \
-chardev socket,id=qmp,path=/var/run/qemu-server/105.qmp,server=on,wait=off  -mon chardev=qmp,mode=control    \
-chardev socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5 -mon chardev=qmp-event,mode=control  -pidfile /var/run/qemu-server/105.pid    \
-daemonize    \
-smbios type=1,uuid=104ec11d-7100-4feb-b953-f058f7e7f622    \
-drive if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//AAVMF_CODE.fd    \
-drive if=pflash,unit=1,format=raw,id=drive-efidisk0,size=67108864,file=/tmp/105-ovmf.fd   \ 
-smp 8,sockets=1,cores=8,maxcpus=8    \
-nodefaults -boot menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg  \  
-device VGA --vnc 0.0.0.0:0   \#### -nographic 改为-device VGA --vnc 0.0.0.0:0  
-cpu cortex-a72 -m 8196    \   
-device nec-usb-xhci -device usb-mouse -device usb-kbd   \ ####添加这行
-device pci-bridge,id=pci.1,chassis_nr=1,bus=pcie.0,addr=0x1e      \  #### 删除这行
-device pci-bridge,id=pci.2,chassis_nr=2,bus=pcie.0,addr=0x1f     \  #### 删除这行
-device usb-ehci,id=ehci,bus=pcie.0,addr=0x1  \  #### 删除这行
-chardev socket,id=serial0,path=/var/run/qemu-server/105.serial0,server=on,wait=off -serial chardev:serial0     \  #### 删除这行
-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x3,free-page-reporting=on -iscsi initiator-name=iqn.1993-08.org.debian:01:d650e5d8b79e     \  #### 删除这行
-device virtio-scsi-pci,id=scsihw0,bus=pcie.0,addr=0x5 -drive file=/mnt/pve/bash/template/iso/Kylin-Server-10-SP2-Release-Build09-20210524-arm64.iso,if=none,id=drive-scsi2,media=cdrom,aio=io_uring   \ 
-device scsi-cd,bus=scsihw0.0,channel=0,scsi-id=0,lun=2,drive=drive-scsi2,id=scsi2,bootindex=100    \
-drive file=/dev/pve/vm-105-disk-1,if=none,id=drive-virtio0,format=raw,cache=none,aio=io_uring,detect-zeroes=on    \
-device virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pcie.0,addr=0xa,bootindex=101    \
-netdev type=tap,id=net0,ifname=tap105i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown    \
-device virtio-net-pci,mac=CA:76:0E:59:4A:B9,netdev=net0,bus=pcie.0,addr=0x12,id=net0,bootindex=102 -machine accel=tcg,type=virt+pve0

5.5 启动vncview 添加链接剩下的和 3.3.1.1之后的图形安装一样速度会快很多

6、安装过程遇到问题

6.1 启动麒麟基本图形（文本）安装界面