ovs-dpdk中有一个配置参数dpdk-lcore-mask ,但使用这个配置参数的时候,主线程的cpu亲和性 限制为master core,导致后续 生成的进程 都只在 master core上运行。
当不配置该参数时,则为自动配置模式,该模式下,最后会 恢复 主线程的cpu亲和性。
推荐配置:不配置 dpdk-lcore-mask 。这样的话,主线程的子线程可在 更多的cpu核中调度。
代码分析
ovs-dpdk中的dpdk-lcore-mask的参数解析代码
dpdk_init__()
construct_dpdk_args(ovs_other_config, &args);
construct_dpdk_options(ovs_other_config, args);
{"dpdk-lcore-mask", "-c", false, NULL},
// if dpdk-lcore-mask is set
if (args_contains(&args, "-c") || args_contains(&args, "-l")) {
auto_determine = false;
}
/**
* NOTE: This is an unsophisticated mechanism for determining the DPDK
* lcore for the DPDK Master.
*/
if (auto_determine) {
const struct ovs_numa_info_core *core;
int cpu = 0;
/* Get the main thread affinity */
affinity = ovs_numa_thread_getaffinity_dump();
if (affinity) {
cpu = INT_MAX;
FOR_EACH_CORE_ON_DUMP (core, affinity) {
if (cpu > core->core_id) {
cpu = core->core_id;
}
}
} else {
/* User did not set dpdk-lcore-mask and unable to get current
* thread affintity - default to core #0 */
VLOG_ERR("Thread getaffinity Failed. Using core #0");
}
svec_add(&args, "-l");
svec_add_nocopy(&args, xasprintf("%d", cpu));
}
// init rte env. dpdk库,会限制主线程的cpu亲和性为master core.
result = rte_eal_init(args.n, argv);
/* Set the main thread affinity back to pre rte_eal_init() value */
// 自动配置模式下(即不配置dpdk-lcore-mask),会恢复cpu亲和性。
if (affinity) {
ovs_numa_thread_setaffinity_dump(affinity);
ovs_numa_dump_destroy(affinity);
}
dpdk库中的亲和性初始化
对 current thread 和 slave work thread的affinity 影响
rte_eal_init()
// 设置当前线程的affinity为master core
pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
&lcore_config[config->main_lcore].cpuset) != 0 {
// 设置work线程的affinity为lcore
RTE_LCORE_FOREACH_WORKER(i) {
eal_thread_create(&lcore_config[i].thread_id, i)
pthread_setaffinity_np(lcore_config[i].thread_id,
sizeof(rte_cpuset_t), &lcore_config[i].cpuset);
}
ovs-dpdk 系统调试
配置1 - 设置dpdk-lcore-mask
ovs-vsctl set Open_vSwitch . other_config:dpdk-lcore-mask="0x00000f0";
ovs-vsctl set Open_vSwitch . other_config:n-handler-threads=8
ovs-vsctl set Open_vSwitch . other_config:n-revalidator-threads=8
log1 - handler固定在core 4
[root@ct8test88 z]# pidstat -t -p $(cat /usr/local/var/run/openvswitch/ovs-vswitchd.pid)
Linux 4.18.0-348.7.1.el8_5.x86_64 (ct8test88) 08/23/2022 _x86_64_ (128 cpu)
07:26:43 PM UID TGID TID %usr %system %guest %wait %cpu cpu Command
07:26:43 PM 0 840428 - 0.01 0.00 0.00 0.00 0.01 4 ovs-vswitchd
07:26:43 PM 0 - 840428 0.00 0.00 0.00 0.00 0.00 4 |__ovs-vswitchd
07:26:43 PM 0 - 840429 0.00 0.00 0.00 0.00 0.00 117 |__eal-intr-thread
07:26:43 PM 0 - 840430 0.00 0.00 0.00 0.00 0.00 119 |__rte_mp_handle
07:26:43 PM 0 - 840431 0.00 0.00 0.00 0.00 0.00 5 |__lcore-worker-5
07:26:43 PM 0 - 840432 0.00 0.00 0.00 0.00 0.00 6 |__lcore-worker-6
07:26:43 PM 0 - 840433 0.00 0.00 0.00 0.00 0.00 7 |__lcore-worker-7
07:26:43 PM 0 - 840438 0.00 0.00 0.00 0.00 0.00 8 |__ovs-vswitchd
07:26:43 PM 0 - 840439 0.00 0.00 0.00 0.00 0.00 4 |__dpdk_watchdog1
07:26:43 PM 0 - 840441 0.00 0.00 0.00 0.00 0.00 4 |__urcu2
07:26:43 PM 0 - 840449 0.00 0.00 0.00 0.00 0.00 4 |__ct_clean3
07:26:43 PM 0 - 840450 0.00 0.00 0.00 0.00 0.00 4 |__ipf_clean14
07:26:43 PM 0 - 840451 0.00 0.00 0.00 0.00 0.00 4 |__handler20
07:26:43 PM 0 - 840452 0.00 0.00 0.00 0.00 0.00 4 |__handler19
07:26:43 PM 0 - 840453 0.00 0.00 0.00 0.00 0.00 4 |__handler18
07:26:43 PM 0 - 840454 0.00 0.00 0.00 0.00 0.00 4 |__handler17
07:26:43 PM 0 - 840455 0.00 0.00 0.00 0.00 0.00 4 |__handler16
07:26:43 PM 0 - 840456 0.00 0.00 0.00 0.00 0.00 4 |__handler15
07:26:43 PM 0 - 840457 0.00 0.00 0.00 0.00 0.00 4 |__handler4
07:26:43 PM 0 - 840458 0.00 0.00 0.00 0.00 0.00 4 |__handler13
07:26:43 PM 0 - 840459 0.00 0.00 0.00 0.00 0.00 4 |__revalidator5
07:26:43 PM 0 - 840460 0.00 0.00 0.00 0.00 0.00 4 |__revalidator6
07:26:43 PM 0 - 840461 0.00 0.00 0.00 0.00 0.00 4 |__revalidator11
07:26:43 PM 0 - 840462 0.00 0.00 0.00 0.00 0.00 4 |__revalidator7
07:26:43 PM 0 - 840463 0.00 0.00 0.00 0.00 0.00 4 |__revalidator8
07:26:43 PM 0 - 840464 0.00 0.00 0.00 0.00 0.00 4 |__revalidator12
07:26:43 PM 0 - 840465 0.00 0.00 0.00 0.00 0.00 4 |__revalidator9
07:26:43 PM 0 - 840466 0.00 0.00 0.00 0.00 0.00 4 |__revalidator10
07:26:43 PM 0 - 840472 0.00 0.00 0.00 0.00 0.00 6 |__pmd-c06/id:24
07:26:43 PM 0 - 840473 0.00 0.00 0.00 0.00 0.00 7 |__pmd-c07/id:23
07:26:43 PM 0 - 840474 0.00 0.00 0.00 0.00 0.00 5 |__pmd-c05/id:22
0
[root@ct8test88 z]# taskset -cp 991962
pid 991962's current affinity list: 4
配置2 - 不设置dpdk-lcore-mask
ovs-vsctl --no-wait set Open_vSwitch . other_config:n-handler-threads=8
ovs-vsctl --no-wait set Open_vSwitch . other_config:n-revalidator-threads=8
log2 - hander 分散在多核中
[root@ct8test88 z]# pidstat -t -p $(cat /usr/local/var/run/openvswitch/ovs-vswitchd.pid)
Linux 4.18.0-348.7.1.el8_5.x86_64 (ct8test88) 08/23/2022 _x86_64_ (128 cpu)
07:34:03 PM UID TGID TID %usr %system %guest %wait %cpu cpu Command
07:34:03 PM 0 841970 - 0.01 0.00 0.00 0.00 0.01 25 ovs-vswitchd
07:34:03 PM 0 - 841970 0.00 0.00 0.00 0.00 0.00 25 |__ovs-vswitchd
07:34:03 PM 0 - 841971 0.00 0.00 0.00 0.00 0.00 121 |__eal-intr-thread
07:34:03 PM 0 - 841972 0.00 0.00 0.00 0.00 0.00 119 |__rte_mp_handle
07:34:03 PM 0 - 841975 0.00 0.00 0.00 0.00 0.00 11 |__ovs-vswitchd
07:34:03 PM 0 - 841976 0.00 0.00 0.00 0.00 0.00 1 |__dpdk_watchdog1
07:34:03 PM 0 - 841978 0.00 0.00 0.00 0.00 0.00 9 |__urcu2
07:34:03 PM 0 - 841988 0.00 0.00 0.00 0.00 0.00 119 |__ct_clean3
07:34:03 PM 0 - 841989 0.00 0.00 0.00 0.00 0.00 122 |__ipf_clean4
07:34:03 PM 0 - 841990 0.00 0.00 0.00 0.00 0.00 60 |__handler5
07:34:03 PM 0 - 841991 0.00 0.00 0.00 0.00 0.00 125 |__handler6
07:34:03 PM 0 - 841992 0.00 0.00 0.00 0.00 0.00 97 |__handler7
07:34:03 PM 0 - 841993 0.00 0.00 0.00 0.00 0.00 98 |__handler8
07:34:03 PM 0 - 841994 0.00 0.00 0.00 0.00 0.00 99 |__handler9
07:34:03 PM 0 - 841995 0.00 0.00 0.00 0.00 0.00 59 |__handler10
07:34:03 PM 0 - 841996 0.00 0.00 0.00 0.00 0.00 100 |__handler11
07:34:03 PM 0 - 841997 0.00 0.00 0.00 0.00 0.00 103 |__handler12
07:34:03 PM 0 - 841998 0.00 0.00 0.00 0.00 0.00 20 |__revalidator13
07:34:03 PM 0 - 841999 0.00 0.00 0.00 0.00 0.00 17 |__revalidator14
07:34:03 PM 0 - 842000 0.00 0.00 0.00 0.00 0.00 14 |__revalidator15
07:34:03 PM 0 - 842001 0.00 0.00 0.00 0.00 0.00 18 |__revalidator16
07:34:03 PM 0 - 842002 0.00 0.00 0.00 0.00 0.00 21 |__revalidator17
07:34:03 PM 0 - 842003 0.00 0.00 0.00 0.00 0.00 24 |__revalidator18
07:34:03 PM 0 - 842004 0.00 0.00 0.00 0.00 0.00 23 |__revalidator19
07:34:03 PM 0 - 842006 0.00 0.00 0.00 0.00 0.00 22 |__revalidator20
07:34:03 PM 0 - 842015 0.00 0.00 0.00 0.00 0.00 6 |__pmd-c06/id:21
07:34:03 PM 0 - 842016 0.00 0.00 0.00 0.00 0.00 7 |__pmd-c07/id:22
07:34:03 PM 0 - 842017 0.00 0.00 0.00 0.00 0.00 5 |__pmd-c05/id:23
07:34:03 PM 0 - 842018 0.00 0.00 0.00 0.00 0.00 4 |__pmd-c04/id:24
[root@ct8test88 z]# taskset -cp 991962
pid 991962's current affinity list: 1-127
参考
OVS-DPDK Parameters: Dealing with multi-NUMA
[ovs-dev] [ovs dpdk] why all the ovs threads pinned to master lcore?
原文地址:https://www.jb51.cc/wenti/3285385.html
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。