【Xilinx】基于MPSoC的OpenAMP实现(二）

前言

参考前文
【Xilinx】基于MPSoC的OpenAMP实现（一）

在上一篇文章里面主要是采用Xilinx提供的现成的openamp.dtsi和已经编译好的image_echo_test固件、echo_test应用程序进行快速测试验证，目的是证明软件环境、硬件环境都能够正常运行OpenAMP。

以此作为基线，我们进一步研究如何修改定制image_echo_test、echo_test、openamp.dtsi，以及内部工作原理。

一、修改echo_test

1、查找源代码

首先我们先找到源代码

xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2$ find . -name echo_test.c
./components/yocto/layers/meta-openamp/recipes-openamp/rpmsg-examples/rpmsg-echo-test/echo_test.c

cd ./components/yocto/layers/meta-openamp/recipes-openamp/rpmsg-examples/
xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/components/yocto/layers/meta-openamp/recipes-openamp/rpmsg-examples$ tree
.
├── rpmsg-echo-test
│   ├── echo_test.c
│   ├── LICENSE
│   └── Makefile
├── rpmsg-echo-test_1.0.bb
├── rpmsg-mat-mul
│   ├── LICENSE
│   ├── Makefile
│   └── mat_mul_demo.c
├── rpmsg-mat-mul_1.0.bb
├── rpmsg-proxy-app
│   ├── LICENSE
│   ├── Makefile
│   ├── proxy_app.c
│   └── proxy_app.h
└── rpmsg-proxy-app_1.0.bb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

可以看到3个测试例子都在components/yocto/layers/meta-openamp/recipes-openamp/rpmsg-examples目录下，代码结构很简单，一个makefie加上一个c文件

Makefile如下

APP = echo_test
APP_OBJS = echo_test.o

# Add any other object files to this list below


all: $(APP)

$(APP): $(APP_OBJS)
	$(CC) $(LDFLAGS) -o $@ $(APP_OBJS) $(LDLIBS)

clean:
	rm -rf $(APP) *.o

%.o: %.c
	$(CC) -c $(CFLAGS) -o $@ $<
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

echo_test.c

/*
 * echo_test.c
 *
 *  Created on: Oct 4, 2014
 *      Author: etsam
 */

/*
 * Test application that data integraty of inter processor
 * communication from linux userspace to a remote software
 * context. The application sends chunks of data to the
 * remote processor. The remote side echoes the data back
 * to application which then validates the data returned.
 */

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

struct _payload {
	unsigned long num;
	unsigned long size;
	char data[];
};

static int charfd = -1, fd = -1, err_cnt;

struct _payload *i_payload;
struct _payload *r_payload;

#define RPMSG_GET_KFIFO_SIZE 1
#define RPMSG_GET_AVAIL_DATA_SIZE 2
#define RPMSG_GET_FREE_SPACE 3

#define RPMSG_HEADER_LEN 16
#define MAX_RPMSG_BUFF_SIZE (512 - RPMSG_HEADER_LEN)
#define PAYLOAD_MIN_SIZE	1
#define PAYLOAD_MAX_SIZE	(MAX_RPMSG_BUFF_SIZE - 24)
#define NUM_PAYLOADS		(PAYLOAD_MAX_SIZE/PAYLOAD_MIN_SIZE)

#define RPMSG_BUS_SYS "/sys/bus/rpmsg"

static int rpmsg_create_ept(int rpfd, struct rpmsg_endpoint_info *eptinfo)
{
	int ret;

	ret = ioctl(rpfd, RPMSG_CREATE_EPT_IOCTL, eptinfo);
	if (ret)
		perror("Failed to create endpoint.\n");
	return ret;
}

static char *get_rpmsg_ept_dev_name(const char *rpmsg_char_name,
				    const char *ept_name,
				    char *ept_dev_name)
{
	char sys_rpmsg_ept_name_path[64];
	char svc_name[64];
	char *sys_rpmsg_path = "/sys/class/rpmsg";
	FILE *fp;
	int i;
	int ept_name_len;

	for (i = 0; i < 128; i++) {
		sprintf(sys_rpmsg_ept_name_path, "%s/%s/rpmsg%d/name",
			sys_rpmsg_path, rpmsg_char_name, i);
		printf("checking %s\n", sys_rpmsg_ept_name_path);
		if (access(sys_rpmsg_ept_name_path, F_OK) < 0)
			continue;
		fp = fopen(sys_rpmsg_ept_name_path, "r");
		if (!fp) {
			printf("failed to open %s\n", sys_rpmsg_ept_name_path);
			break;
		}
		fgets(svc_name, sizeof(svc_name), fp);
		fclose(fp);
		printf("svc_name: %s.\n",svc_name);
		ept_name_len = strlen(ept_name);
		if (ept_name_len > sizeof(svc_name))
			ept_name_len = sizeof(svc_name);
		if (!strncmp(svc_name, ept_name, ept_name_len)) {
			sprintf(ept_dev_name, "rpmsg%d", i);
			return ept_dev_name;
		}
	}

	printf("Not able to RPMsg endpoint file for %s:%s.\n",
	       rpmsg_char_name, ept_name);
	return NULL;
}

static int bind_rpmsg_chrdev(const char *rpmsg_dev_name)
{
	char fpath[256];
	char *rpmsg_chdrv = "rpmsg_chrdev";
	int fd;
	int ret;


	/* rpmsg dev overrides path */
	sprintf(fpath, "%s/devices/%s/driver_override",
		RPMSG_BUS_SYS, rpmsg_dev_name);
	fd = open(fpath, O_WRONLY);
	if (fd < 0) {
		fprintf(stderr, "Failed to open %s, %s\n",
			fpath, strerror(errno));
		return -EINVAL;
	}
	ret = write(fd, rpmsg_chdrv, strlen(rpmsg_chdrv) + 1);
	if (ret < 0) {
		fprintf(stderr, "Failed to write %s to %s, %s\n",
			rpmsg_chdrv, fpath, strerror(errno));
		return -EINVAL;
	}
	close(fd);

	/* bind the rpmsg device to rpmsg char driver */
	sprintf(fpath, "%s/drivers/%s/bind", RPMSG_BUS_SYS, rpmsg_chdrv);
	fd = open(fpath, O_WRONLY);
	if (fd < 0) {
		fprintf(stderr, "Failed to open %s, %s\n",
			fpath, strerror(errno));
		return -EINVAL;
	}
	ret = write(fd, rpmsg_dev_name, strlen(rpmsg_dev_name) + 1);
	if (ret < 0) {
		fprintf(stderr, "Failed to write %s to %s, %s\n",
			rpmsg_dev_name, fpath, strerror(errno));
		return -EINVAL;
	}
	close(fd);
	return 0;
}

static int get_rpmsg_chrdev_fd(const char *rpmsg_dev_name,
			       char *rpmsg_ctrl_name)
{
	char dpath[256];
	char fpath[256];
	char *rpmsg_ctrl_prefix = "rpmsg_ctrl";
	DIR *dir;
	struct dirent *ent;
	int fd;

	sprintf(dpath, "%s/devices/%s/rpmsg", RPMSG_BUS_SYS, rpmsg_dev_name);
	dir = opendir(dpath);
	if (dir == NULL) {
		fprintf(stderr, "Failed to open dir %s\n", dpath);
		return -EINVAL;
	}
	while ((ent = readdir(dir)) != NULL) {
		if (!strncmp(ent->d_name, rpmsg_ctrl_prefix,
			    strlen(rpmsg_ctrl_prefix))) {
			printf("Opening file %s.\n", ent->d_name);
			sprintf(fpath, "/dev/%s", ent->d_name);
			fd = open(fpath, O_RDWR | O_NONBLOCK);
			if (fd < 0) {
				fprintf(stderr,
					"Failed to open rpmsg char dev %s,%s\n",
					fpath, strerror(errno));
				return fd;
			}
			sprintf(rpmsg_ctrl_name, "%s", ent->d_name);
			return fd;
		}
	}

	fprintf(stderr, "No rpmsg char dev file is found\n");
	return -EINVAL;
}

int main(int argc, char *argv[])
{
	int ret, i, j;
	int size, bytes_rcvd, bytes_sent;
	err_cnt = 0;
	int opt;
	char *rpmsg_dev="virtio0.rpmsg-openamp-demo-channel.-1.0";
	int ntimes = 1;
	char fpath[256];
	char rpmsg_char_name[16];
	struct rpmsg_endpoint_info eptinfo;
	char ept_dev_name[16];
	char ept_dev_path[32];


	while ((opt = getopt(argc, argv, "d:n:")) != -1) {
		switch (opt) {
		case 'd':
			rpmsg_dev = optarg;
			break;
		case 'n':
			ntimes = atoi(optarg);
			break;
		default:
			printf("getopt return unsupported option: -%c\n",opt);
			break;
		}
	}
	printf("\r\n Echo test start \r\n");

	/* Load rpmsg_char driver */
	printf("\r\nMaster>probe rpmsg_char\r\n");
	ret = system("modprobe rpmsg_char");
	if (ret < 0) {
		perror("Failed to load rpmsg_char driver.\n");
		return -EINVAL;
	}

	printf("\r\n Open rpmsg dev %s! \r\n", rpmsg_dev);
	sprintf(fpath, "%s/devices/%s", RPMSG_BUS_SYS, rpmsg_dev);
	if (access(fpath, F_OK)) {
		fprintf(stderr, "Not able to access rpmsg device %s, %s\n",
			fpath, strerror(errno));
		return -EINVAL;
	}
	ret = bind_rpmsg_chrdev(rpmsg_dev);
	if (ret < 0)
		return ret;
	charfd = get_rpmsg_chrdev_fd(rpmsg_dev, rpmsg_char_name);
	if (charfd < 0)
		return charfd;

	/* Create endpoint from rpmsg char driver */
	strcpy(eptinfo.name, "rpmsg-openamp-demo-channel");
	eptinfo.src = 0;
	eptinfo.dst = 0xFFFFFFFF;
	ret = rpmsg_create_ept(charfd, &eptinfo);
	if (ret) {
		printf("failed to create RPMsg endpoint.\n");
		return -EINVAL;
	}
	if (!get_rpmsg_ept_dev_name(rpmsg_char_name, eptinfo.name,
				    ept_dev_name))
		return -EINVAL;
	sprintf(ept_dev_path, "/dev/%s", ept_dev_name);
	fd = open(ept_dev_path, O_RDWR | O_NONBLOCK);
	if (fd < 0) {
		perror("Failed to open rpmsg device.");
		close(charfd);
		return -1;
	}

	i_payload = (struct _payload *)malloc(2 * sizeof(unsigned long) + PAYLOAD_MAX_SIZE);
	r_payload = (struct _payload *)malloc(2 * sizeof(unsigned long) + PAYLOAD_MAX_SIZE);

	if (i_payload == 0 || r_payload == 0) {
		printf("ERROR: Failed to allocate memory for payload.\n");
		return -1;
	}

	for (j=0; j < ntimes; j++){
		printf("\r\n **********************************");
		printf("****\r\n");
		printf("\r\n  Echo Test Round %d \r\n", j);
		printf("\r\n **********************************");
		printf("****\r\n");
		for (i = 0, size = PAYLOAD_MIN_SIZE; i < NUM_PAYLOADS;
		i++, size++) {
			int k;

			i_payload->num = i;
			i_payload->size = size;

			/* Mark the data buffer. */
			memset(&(i_payload->data[0]), 0xA5, size);

			printf("\r\n sending payload number");
			printf(" %ld of size %d\r\n", i_payload->num,
			(2 * sizeof(unsigned long)) + size);

			bytes_sent = write(fd, i_payload,
			(2 * sizeof(unsigned long)) + size);

			if (bytes_sent <= 0) {
				printf("\r\n Error sending data");
				printf(" .. \r\n");
				break;
			}
			printf("echo test: sent : %d\n", bytes_sent);

			r_payload->num = 0;
			bytes_rcvd = read(fd, r_payload,
					(2 * sizeof(unsigned long)) + PAYLOAD_MAX_SIZE);
			while (bytes_rcvd <= 0) {
				usleep(10000);
				bytes_rcvd = read(fd, r_payload,
					(2 * sizeof(unsigned long)) + PAYLOAD_MAX_SIZE);
			}
			printf(" received payload number ");
			printf("%ld of size %d\r\n", r_payload->num, bytes_rcvd);

			/* Validate data buffer integrity. */
			for (k = 0; k < r_payload->size; k++) {

				if (r_payload->data[k] != 0xA5) {
					printf(" \r\n Data corruption");
					printf(" at index %d \r\n", k);
					err_cnt++;
					break;
				}
			}
			bytes_rcvd = read(fd, r_payload,
			(2 * sizeof(unsigned long)) + PAYLOAD_MAX_SIZE);

		}
		printf("\r\n **********************************");
		printf("****\r\n");
		printf("\r\n Echo Test Round %d Test Results: Error count = %d\r\n",
		j, err_cnt);
		printf("\r\n **********************************");
		printf("****\r\n");
	}

	free(i_payload);
	free(r_payload);

	close(fd);
	if (charfd >= 0)
		close(charfd);
	return 0;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330

二、创建自己的app

1、新建app

进入petalinux工程主目录，运行下面命令，-n后面是你项目的名称

petalinux-create -t apps -n leo-echo-test --enable
1

运行完成，会在project-spec/meta-user/recipes-apps创建了leo-echo-test目录

xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/project-spec/meta-user/recipes-apps/leo-echo-test$ tree
.
├── files
│   ├── leo-echo-test.c
│   └── Makefile
├── leo-echo-test.bb
└── README

1 directory, 4 files
1
2
3
4
5
6
7
8
9

2、 编译代码

把echo-test.c的代码全选复制到leo-echo-test.c，然后编译

2.1 用petalinux-build编译

petalinux-build -c leo-echo-test

petalinux-build -c leo-echo-test -x do_install
petalinux-build -c rootfs
petalinux-build -x package
1
2
3
4
5

但是这种方式需要遍历很多东西，编译非常慢，每次需要几分钟

2.2 方法二：创建sdk，用sdk编译（重点）

2.2.1 编译并安装sdk

编译sdk需要时间比较长，但是强烈建议不要用网上下载的其他编译器，可能会有未知的兼容问题。

time petalinux-build --sdk
time petalinux-package --sysroot
1
2

我安装完成后sdk的完整路径是

/opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk

库文件的搜索路径是
/opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk/sysroots/aarch64-xilinx-linux/usr/include

2.2.2 用sdk编译应用程序

使用前要先初始化sdk的环境变量
source运行environment-setup-aarch64-xilinx-linux配置

source /opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk/environment-setup-aarch64-xilinx-linux
1

有可能会提示下面的错误

Your environment is misconfigured, you probably need to 'unset LD_LIBRARY_PATH'
but please check why this was set in the first place and that it's safe to unset.
The SDK will not operate correctly in most cases when LD_LIBRARY_PATH is set.
For more references see:
  http://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html#AEN80
  http://xahlee.info/UnixResource_dir/_/ldpath.html
1
2
3
4
5
6

查看$LD_LIBRARY_PATH变量，非空的话用unset LD_LIBRARY_PATH取消掉。我是因为装有qt的原因。

#查看LD_LIBRARY_PATH
echo $LD_LIBRARY_PATH
/opt/work/Qt5.13.2/Tools/QtCreator/lib:

#删除LD_LIBRARY_PATH
unset LD_LIBRARY_PATH
1
2
3
4
5
6

然后就可以用make来编译应用程序了

xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/project-spec/meta-user/recipes-apps/leo-echo-test/files$ make
aarch64-xilinx-linux-gcc  -march=armv8-a+crc -mtune=cortex-a72.cortex-a53 --sysroot=/opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk/sysroots/aarch64-xilinx-linux  -O2 -pipe -g -feliminate-unused-debug-types    -c -o leo-echo-test.o leo-echo-test.c
aarch64-xilinx-linux-gcc  -march=armv8-a+crc -mtune=cortex-a72.cortex-a53 --sysroot=/opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk/sysroots/aarch64-xilinx-linux -o leo-echo-test leo-echo-test.o -Wl,-O1 -Wl,--hash-style=gnu -Wl,--as-needed 


#用ll或者ls -l查看，已经生成了leo-echo-test
xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/project-spec/meta-user/recipes-apps/leo-echo-test/files$ ls -l
total 100
-rwxrwxr-x 1 xlx xlx 37208 9月  19 07:11 leo-echo-test
-rw-rw-r-- 1 xlx xlx  8319 9月  18 23:01 leo-echo-test.c
-rw-rw-r-- 1 xlx xlx 44576 9月  19 07:11 leo-echo-test.o
-rw-rw-r-- 1 xlx xlx   216 9月  18 22:56 Makefile

#用readelf命令可以查看文件格式，可以看到是AArch64
xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/project-spec/meta-user/recipes-apps/leo-echo-test/files$ readelf -h leo-echo-test
ELF Header:
  Magic:   7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 
  Class:                             ELF64
  Data:                              2's complement, little endian
  Version:                           1 (current)
  OS/ABI:                            UNIX - System V
  ABI Version:                       0
  Type:                              DYN (Shared object file)
  Machine:                           AArch64
  Version:                           0x1
  Entry point address:               0x17e4
  Start of program headers:          64 (bytes into file)
  Start of section headers:          34904 (bytes into file)
  Flags:                             0x0
  Size of this header:               64 (bytes)
  Size of program headers:           56 (bytes)
  Number of program headers:         9
  Size of section headers:           64 (bytes)
  Number of section headers:         36
  Section header string table index: 35
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

建议用nfs把开发目录映射到开发板上，这样编译完成可以直接在开发板上运行。避免用sd卡或者tftp，需要反复复制下载

详细请参考下面链接
Petalinux快速入门向导 (11) 第十章.配置nfs服务器 https://blog.csdn.net/aatu/article/details/120755209

三、需要注意的地方

ret = system(“modprobe rpmsg_char”);

/sys/devices/platform/ff9a0000.zynqmp-rpu/r5@0/r5@0#vdev0buffer/virtio0/virtio0.rpmsg-openamp-demo-channel.-1.0

/sys/bus/rpmsg/devices/virtio0.rpmsg-openamp-demo-channel.-1.0 -> …/…/…/devices/platform/ff9a0000.zynqmp-rpu/r5@0/r5@0#vdev0buffer/virtio0/virtio0.rpmsg-openamp-demo-channel.-1.0

创建链接
ln -s /sys/devices/platform/ff9a0000.zynqmp-rpu/r5@0/r5@0#vdev0buffer/virtio0/virtio0.rpmsg-openamp-demo-channel.-1.0 /sys/bus/rpmsg/devices/virtio0.rpmsg-openamp-demo-channel.-1.0

Echo test start --2023-9-18 By Leo

Master>probe rpmsg_char

1Open rpmsg dev [/sys/bus/rpmsg/devices/virtio0.rpmsg-openamp-demo-channel.-1.0]!
Failed to write virtio0.rpmsg-openamp-demo-channel.-1.0 to /sys/bus/rpmsg/drivers/rpmsg_chrdev/bind, No such device

root@xilinx-zcu106-2020_2:/mnt/nfs# ls /sys/bus/rpmsg/drivers/rpmsg_chrdev/ -l
total 0
–w------- 1 root root 4096 Sep 19 01:00 bind
lrwxrwxrwx 1 root root 0 Sep 19 01:08 module -> …/…/…/…/module/rpmsg_char
–w------- 1 root root 4096 Sep 19 01:00 uevent
–w------- 1 root root 4096 Sep 19 01:08 unbind
lrwxrwxrwx 1 root root 0 Sep 19 01:08 virtio0.rpmsg-openamp-demo-channel.-1.0 -> …/…/…/…/devices/platform/ff9a0000.zynqmp-rpu/r5@0/r5@0#vdev0buffer/virtio0/virtio0.rpmsg-openamp-demo-channel.-1.0

下一步讲如何修改image_echo_test。
未完待续