external/srs
1
0
Fork 0
mirror of https://github.com/ossrs/srs.git synced 2025-03-09 15:49:59 +00:00
Code Issues Releases Wiki Activity

Refine research zerocopy, use batch for reception

Browse source
This commit is contained in:
winlin 2020-04-20 21:18:49 +08:00
parent 72322836c6
commit 74416e476a
1 changed files with 107 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

106
trunk/research/msg_zerocopy/client.cpp
View file

@ -34,6 +34,7 @@ void* receiver(void* arg)
{ {
st_netfd_t stfd = (st_netfd_t)arg; st_netfd_t stfd = (st_netfd_t)arg;
for (;;) {
sockaddr_in peer; sockaddr_in peer;
memset(&peer, 0, sizeof(sockaddr_in)); memset(&peer, 0, sizeof(sockaddr_in));
@ -55,16 +56,56 @@ void* receiver(void* arg)
assert(r0 > 0); assert(r0 > 0);
printf("Pong %s:%d %d bytes, flags %#x, %s\n", inet_ntoa(peer.sin_addr), ntohs(peer.sin_port), r0, printf("Pong %s:%d %d bytes, flags %#x, %s\n", inet_ntoa(peer.sin_addr), ntohs(peer.sin_port), r0,
msg.msg_flags, msg.msg_iov->iov_base); msg.msg_flags, msg.msg_iov->iov_base);
}
return NULL; return NULL;
} }
void parse_reception(st_netfd_t stfd)
{
msghdr msg;
memset(&msg, 0, sizeof(msghdr));
// Reception from kernel, @see https://www.kernel.org/doc/html/latest/networking/msg_zerocopy.html#notification-reception
// See do_recv_completion at https://github.com/torvalds/linux/blob/master/tools/testing/selftests/net/msg_zerocopy.c#L393
char control[100];
msg.msg_control = control;
msg.msg_controllen = sizeof(control);
// Note that the r0 is 0, the reception is in the control.
int r0 = st_recvmsg(stfd, &msg, MSG_ERRQUEUE, ST_UTIME_NO_TIMEOUT);
assert(r0 >= 0);
assert(msg.msg_flags == MSG_ERRQUEUE);
// Notification parsing, @see https://www.kernel.org/doc/html/latest/networking/msg_zerocopy.html#notification-parsing
cmsghdr* cm = CMSG_FIRSTHDR(&msg);
assert(cm->cmsg_level == SOL_IP || cm->cmsg_type == IP_RECVERR);
sock_extended_err* serr = (sock_extended_err*)(void*)CMSG_DATA(cm);
assert(serr->ee_errno == 0 && serr->ee_origin == SO_EE_ORIGIN_ZEROCOPY);
uint32_t hi = serr->ee_data;
uint32_t lo = serr->ee_info;
uint32_t range = hi - lo + 1;
printf("Reception %d bytes, flags %#x, cmsg(level %#x, type %#x), serr(errno %#x, origin %#x, code %#x), range %d [%d, %d]\n",
msg.msg_controllen, msg.msg_flags, cm->cmsg_level, cm->cmsg_type, serr->ee_errno, serr->ee_origin, serr->ee_code, range, lo, hi);
// Defered Copies, @see https://www.kernel.org/doc/html/latest/networking/msg_zerocopy.html#deferred-copies
if (serr->ee_code == SO_EE_CODE_ZEROCOPY_COPIED) {
printf("Warning: Defered copies, should stop zerocopy\n");
}
}
int main(int argc, char** argv) int main(int argc, char** argv)
{ {
if (argc < 5) { if (argc < 8) {
printf("Usage: %s <host> <port> <pong> <zerocopy>\n", argv[0]); printf("Usage: %s <host> <port> <pong> <zerocopy> <sendmmsg> <loop> <batch>\n", argv[0]);
printf(" pong Whether response pong, true|false\n");
printf(" zerocopy Whether use zerocopy to sendmsg, true|false\n");
printf(" sendmmsg The copies of message, 1 means sendmmsg(msg+msg)\n");
printf(" loop The number of loop to send out messages\n");
printf(" batch Whether read reception by batch, true|false\n");
printf("For example:\n"); printf("For example:\n");
printf(" %s 127.0.0.1 8000 true true\n", argv[0]); printf(" %s 127.0.0.1 8000 true true 0 1 true\n", argv[0]);
exit(-1); exit(-1);
} }
@ -72,7 +113,11 @@ int main(int argc, char** argv)
int port = atoi(argv[2]); int port = atoi(argv[2]);
bool pong = !strcmp(argv[3], "true"); bool pong = !strcmp(argv[3], "true");
bool zerocopy = !strcmp(argv[4], "true"); bool zerocopy = !strcmp(argv[4], "true");
printf("Server listen %s:%d, pong %d, zerocopy %d\n", host, port, pong, zerocopy); int nn_copies = atoi(argv[5]);
int loop = atoi(argv[6]);
bool batch = !strcmp(argv[7], "true");
printf("Server listen %s:%d, pong %d, zerocopy %d, copies %d, loop %d, batch %d\n",
host, port, pong, zerocopy, nn_copies, loop, batch);
assert(!st_set_eventsys(ST_EVENTSYS_ALT)); assert(!st_set_eventsys(ST_EVENTSYS_ALT));
assert(!st_init()); assert(!st_init());
@ -121,6 +166,7 @@ int main(int argc, char** argv)
iov.iov_base = buf; iov.iov_base = buf;
iov.iov_len = strlen(buf); iov.iov_len = strlen(buf);
for (int k = 0; k < loop; k++) {
msghdr msg; msghdr msg;
memset(&msg, 0, sizeof(msghdr)); memset(&msg, 0, sizeof(msghdr));
msg.msg_name = (sockaddr_in*)&peer; msg.msg_name = (sockaddr_in*)&peer;
@ -129,43 +175,43 @@ int main(int argc, char** argv)
msg.msg_iovlen = 1; msg.msg_iovlen = 1;
int r0; int r0;
if (nn_copies == 0) {
if (zerocopy) { if (zerocopy) {
r0 = st_sendmsg(stfd, &msg, MSG_ZEROCOPY, ST_UTIME_NO_TIMEOUT); r0 = st_sendmsg(stfd, &msg, MSG_ZEROCOPY, ST_UTIME_NO_TIMEOUT);
} else { } else {
r0 = st_sendmsg(stfd, &msg, 0, ST_UTIME_NO_TIMEOUT); r0 = st_sendmsg(stfd, &msg, 0, ST_UTIME_NO_TIMEOUT);
} }
} else {
mmsghdr* hdrs = new mmsghdr[nn_copies + 1];
for (int i = 0; i < nn_copies + 1; i++) {
mmsghdr* p = hdrs + i;
memcpy(&p->msg_hdr, &msg, sizeof(msghdr));
p->msg_len = 0;
}
if (zerocopy) {
r0 = st_sendmmsg(stfd, hdrs, nn_copies + 1, MSG_ZEROCOPY, ST_UTIME_NO_TIMEOUT);
} else {
r0 = st_sendmmsg(stfd, hdrs, nn_copies + 1, 0, ST_UTIME_NO_TIMEOUT);
}
}
assert(r0 > 0); assert(r0 > 0);
printf("Ping %s:%d %d bytes, r0=%d, %s\n", host, port, iov.iov_len, r0, msg.msg_iov->iov_base); printf("Ping %s:%d %d bytes, copies=%d, r0=%d, %s\n", host, port, iov.iov_len, nn_copies, r0, msg.msg_iov->iov_base);
// Reception from kernel, @see https://www.kernel.org/doc/html/latest/networking/msg_zerocopy.html#notification-reception if (!zerocopy) {
// See do_recv_completion at https://github.com/torvalds/linux/blob/master/tools/testing/selftests/net/msg_zerocopy.c#L393 continue;
char control[100]; }
msg.msg_control = control;
msg.msg_controllen = sizeof(control);
// Note that the r0 is 0, the reception is in the control.
r0 = st_recvmsg(stfd, &msg, MSG_ERRQUEUE, ST_UTIME_NO_TIMEOUT);
assert(r0 >= 0);
assert(msg.msg_flags == MSG_ERRQUEUE);
// Notification parsing, @see https://www.kernel.org/doc/html/latest/networking/msg_zerocopy.html#notification-parsing if (!batch) {
cmsghdr* cm = CMSG_FIRSTHDR(&msg); parse_reception(stfd);
assert(cm->cmsg_level == SOL_IP || cm->cmsg_type == IP_RECVERR); }
}
sock_extended_err* serr = (sock_extended_err*)(void*)CMSG_DATA(cm); // @see https://www.kernel.org/doc/html/latest/networking/msg_zerocopy.html#notification-batching
assert(serr->ee_errno == 0 && serr->ee_origin == SO_EE_ORIGIN_ZEROCOPY); if (batch) {
st_usleep(100 * 1000);
uint32_t hi = serr->ee_data; parse_reception(stfd);
uint32_t lo = serr->ee_info;
uint32_t range = hi - lo + 1;
printf("Reception %d bytes, flags %#x, cmsg(level %#x, type %#x), serr(errno %#x, origin %#x, code %#x), %d [%d, %d]\n",
msg.msg_controllen, msg.msg_flags, cm->cmsg_level, cm->cmsg_type, serr->ee_errno, serr->ee_origin, serr->ee_code, range, lo, hi);
// Defered Copies, @see https://www.kernel.org/doc/html/latest/networking/msg_zerocopy.html#deferred-copies
if (serr->ee_code == SO_EE_CODE_ZEROCOPY_COPIED) {
printf("Warning: Defered copies, should stop zerocopy\n");
} }
st_sleep(-1); st_sleep(-1);
return 0; return 0;
} }