From a3dc56c01461bdffcac8193ead654ae79225220f Mon Sep 17 00:00:00 2001 From: Pim van Pelt Date: Fri, 13 Aug 2021 14:33:03 +0200 Subject: [PATCH] Force a sync of interfaces when VPP phy changes state When Linux changes link on a master interface, all of its children also change. This is not true in VPP, where bringing down a phy while its sub-ints are up will not change link on the sub-ints. We are forced to undo that change by walking the sub-interfaces of a phy and syncing their state back into linux. For simplicity, just walk all interfaces, as others will be a no-op. The approach may have to be revisited once netlink messages are consumed, to ensure there is not an oscillation where netlink sets a link, which forces all links to be reset, generating more netlink messages, etc. Care should be taken when netlink consumption comes into play! DBGvpp# create sub TenGigabitEthernet3/0/0 1 DBGvpp# lcp create TenGigabitEthernet3/0/0 host-if e0 DBGvpp# lcp create TenGigabitEthernet3/0/0.1 host-if e0.1 593: e0: mtu 9000 qdisc mq state DOWN mode DEFAULT group default qlen 1000 594: e0.1@e0: mtu 9000 qdisc noop state DOWN mode DEFAULT group default qlen 1000 DBGvpp# set int state TenGigabitEthernet3/0/0 up 593: e0: mtu 9000 qdisc mq state UNKNOWN mode DEFAULT group default qlen 1000 594: e0.1@e0: mtu 9000 qdisc noqueue state DOWN mode DEFAULT group default qlen 1000 DBGvpp# set int state TenGigabitEthernet3/0/0.1 up 593: e0: mtu 9000 qdisc mq state UNKNOWN mode DEFAULT group default qlen 1000 594: e0.1@e0: mtu 9000 qdisc noqueue state UP mode DEFAULT group default qlen 1000 DBGvpp# set int state TenGigabitEthernet3/0/0 down 593: e0: mtu 9000 qdisc mq state DOWN mode DEFAULT group default qlen 1000 594: e0.1@e0: mtu 9000 qdisc noqueue state DOWN mode DEFAULT group default qlen 1000 DBGvpp# set int state TenGigabitEthernet3/0/0 up DBGvpp# set int state TenGigabitEthernet3/0/0.1 down 593: e0: mtu 9000 qdisc mq state UP mode DEFAULT group default qlen 1000 594: e0.1@e0: mtu 9000 qdisc noqueue state DOWN mode DEFAULT group default qlen 1000 DBGvpp# set int state TenGigabitEthernet3/0/0 down 593: e0: mtu 9000 qdisc mq state DOWN mode DEFAULT group default qlen 1000 594: e0.1@e0: mtu 9000 qdisc noqueue state DOWN mode DEFAULT group default qlen 1000 --- lcpng_if_sync.c | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/lcpng_if_sync.c b/lcpng_if_sync.c index 5389bfb..3d41200 100644 --- a/lcpng_if_sync.c +++ b/lcpng_if_sync.c @@ -32,6 +32,29 @@ #include #include +/* walk function to copy forward all sw interface link state flags into + * their counterpart LIP link state. + */ +static walk_rc_t +lcp_itf_pair_walk_sync_state_cb (index_t lipi, void *ctx) +{ + lcp_itf_pair_t *lip; + vnet_sw_interface_t *phy; + + lip = lcp_itf_pair_get (lipi); + if (!lip) + return WALK_CONTINUE; + + phy = vnet_get_sw_interface_or_null (vnet_get_main(), lip->lip_phy_sw_if_index); + if (!phy) return WALK_CONTINUE; + + LCP_ITF_PAIR_DBG ("walk_sync_state: lip %U flags %u", + format_lcp_itf_pair, lip, + phy->flags); + lcp_itf_set_link_state (lip, (phy->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)); + + return WALK_CONTINUE; +} static clib_error_t * lcp_itf_admin_state_change (vnet_main_t * vnm, u32 sw_if_index, u32 flags) @@ -49,23 +72,24 @@ lcp_itf_admin_state_change (vnet_main_t * vnm, u32 sw_if_index, u32 flags) if (!lip) return NULL; LCP_ITF_PAIR_INFO ("admin_state_change: %U flags %u", format_lcp_itf_pair, lip, flags); - lcp_itf_set_link_state (lip, (flags & VNET_HW_INTERFACE_FLAG_LINK_UP)); + lcp_itf_set_link_state (lip, (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)); // Sync PHY carrier changes into TAP hi = vnet_get_hw_interface_or_null (vnm, sw_if_index); si = vnet_get_sw_interface_or_null (vnm, lip->lip_host_sw_if_index); if (!si || !hi) return NULL; - LCP_ITF_PAIR_INFO ("admin_state_change: hi %U si %U %u", + LCP_ITF_PAIR_DBG ("admin_state_change: hi %U si %U %u", format_vnet_sw_if_index_name, vnm, hi->hw_if_index, format_vnet_sw_if_index_name, vnm, si->sw_if_index, flags); + tap_set_carrier (si->hw_if_index, (hi->flags & VNET_HW_INTERFACE_FLAG_LINK_UP)); + + // When Linux changes link on a master interface, all of its children also change. + // This is not true in VPP, so we are forced to undo that change by walking the + // sub-interfaces of a phy and syncing their state back into linux. + // For simplicity, just walk all interfaces. + lcp_itf_pair_walk (lcp_itf_pair_walk_sync_state_cb, 0); - tap_set_carrier (si->hw_if_index, (flags & VNET_HW_INTERFACE_FLAG_LINK_UP)); - /* TODO(pim): is this right? see tap_set_speed() "Cannot open netns" error - if (flags & VNET_HW_INTERFACE_FLAG_LINK_UP) { - tap_set_speed (si->hw_if_index, hi->link_speed); - } - */ return NULL; }