controller: Announce routes via route-exchange.

This engine node takes the routes from the "route" engine node and ensures
they are written to the linux side.

It is separate from the "route" engine node as it will also be used to
learn routes in the future.

Signed-off-by: Felix Huettner <felix.huettner@stackit.cloud>
Signed-off-by: 0-day Robot <robot@bytheb.org>

controller/automake.mk

@@ -52,13 +52,18 @@ controller_ovn_controller_SOURCES = \
 	controller/ct-zone.c \
 	controller/ovn-dns.c \
 	controller/ovn-dns.h \
+	controller/route-exchange.h \
 	controller/route.h \
 	controller/route.c
 
 if HAVE_NETLINK
 controller_ovn_controller_SOURCES += \
 	controller/route-exchange-netlink.h \
-	controller/route-exchange-netlink.c
+	controller/route-exchange-netlink.c \
+	controller/route-exchange.c
+else
+controller_ovn_controller_SOURCES += \
+	controller/route-exchange-stub.c
 endif
 
 controller_ovn_controller_LDADD = lib/libovn.la $(OVS_LIBDIR)/libopenvswitch.la

controller/ovn-controller.c

@@ -89,6 +89,7 @@
 #include "ct-zone.h"
 #include "ovn-dns.h"
 #include "route.h"
+#include "route-exchange.h"
 
 VLOG_DEFINE_THIS_MODULE(main);
 
@@ -232,6 +233,8 @@ update_sb_monitors(struct ovsdb_idl *ovnsb_idl,
      *
      * Monitor Template_Var for local chassis.
      *
+     * Monitor Advertised_Route for local datapaths.
+     *
      * We always monitor patch ports because they allow us to see the linkages
      * between related logical datapaths.  That way, when we know that we have
      * a VIF on a particular logical switch, we immediately know to monitor all
@@ -248,6 +251,7 @@ update_sb_monitors(struct ovsdb_idl *ovnsb_idl,
     struct ovsdb_idl_condition igmp = OVSDB_IDL_CONDITION_INIT(&igmp);
     struct ovsdb_idl_condition chprv = OVSDB_IDL_CONDITION_INIT(&chprv);
     struct ovsdb_idl_condition tv = OVSDB_IDL_CONDITION_INIT(&tv);
+    struct ovsdb_idl_condition ar = OVSDB_IDL_CONDITION_INIT(&ar);
 
     /* Always monitor all logical datapath groups. Otherwise, DPG updates may
      * be received *after* the lflows using it are seen by ovn-controller.
@@ -267,6 +271,7 @@ update_sb_monitors(struct ovsdb_idl *ovnsb_idl,
         ovsdb_idl_condition_add_clause_true(&igmp);
         ovsdb_idl_condition_add_clause_true(&chprv);
         ovsdb_idl_condition_add_clause_true(&tv);
+        ovsdb_idl_condition_add_clause_true(&ar);
         goto out;
     }
 
@@ -355,6 +360,7 @@ update_sb_monitors(struct ovsdb_idl *ovnsb_idl,
             sbrec_dns_add_clause_datapaths(&dns, OVSDB_F_INCLUDES, &uuid, 1);
             sbrec_ip_multicast_add_clause_datapath(&ip_mcast, OVSDB_F_EQ,
                                                    uuid);
+            sbrec_advertised_route_add_clause_datapath(&ar, OVSDB_F_EQ, uuid);
         }
 
         /* Datapath groups are immutable, which means a new group record is
@@ -382,6 +388,7 @@ out:;
         sb_table_set_req_mon_condition(ovnsb_idl, igmp_group, &igmp),
         sb_table_set_req_mon_condition(ovnsb_idl, chassis_private, &chprv),
         sb_table_set_opt_mon_condition(ovnsb_idl, chassis_template_var, &tv),
+        sb_table_set_opt_mon_condition(ovnsb_idl, advertised_route, &ar),
     };
 
     unsigned int expected_cond_seqno = 0;
@@ -401,6 +408,7 @@ out:;
     ovsdb_idl_condition_destroy(&igmp);
     ovsdb_idl_condition_destroy(&chprv);
     ovsdb_idl_condition_destroy(&tv);
+    ovsdb_idl_condition_destroy(&ar);
     return expected_cond_seqno;
 }
 
@@ -4785,6 +4793,14 @@ controller_output_bfd_chassis_handler(struct engine_node *node,
     return true;
 }
 
+static bool
+controller_output_route_exchange_handler(struct engine_node *node,
+                                         void *data OVS_UNUSED)
+{
+    engine_set_node_state(node, EN_UPDATED);
+    return true;
+}
+
 /* Handles sbrec_chassis changes.
  * If a new chassis is added or removed return false, so that
  * flows are recomputed.  For any updates, there is no need for
@@ -4977,6 +4993,36 @@ route_sb_advertised_route_data_handler(struct engine_node *node, void *data)
     return true;
 }
 
+static void
+en_route_exchange_run(struct engine_node *node, void *data OVS_UNUSED)
+{
+    struct ed_type_route *route_data =
+        engine_get_input_data("route", node);
+
+    struct route_exchange_ctx_in r_ctx_in = {
+        .announce_routes = &route_data->announce_routes,
+    };
+
+    struct route_exchange_ctx_out r_ctx_out = {
+    };
+
+    route_exchange_run(&r_ctx_in, &r_ctx_out);
+
+    engine_set_node_state(node, EN_UPDATED);
+}
+
+
+static void *
+en_route_exchange_init(struct engine_node *node OVS_UNUSED,
+                       struct engine_arg *arg OVS_UNUSED)
+{
+    return NULL;
+}
+
+static void
+en_route_exchange_cleanup(void *data OVS_UNUSED)
+{}
+
 /* Returns false if the northd internal version stored in SB_Global
  * and ovn-controller internal version don't match.
  */
@@ -5212,6 +5258,8 @@ main(int argc, char *argv[])
     ovsdb_idl_omit(ovnsb_idl_loop.idl, &sbrec_ha_chassis_col_external_ids);
     ovsdb_idl_omit(ovnsb_idl_loop.idl,
                    &sbrec_ha_chassis_group_col_external_ids);
+    ovsdb_idl_omit(ovnsb_idl_loop.idl,
+                   &sbrec_advertised_route_col_external_ids);
 
     /* We don't want to monitor Connection table at all. So omit all the
      * columns. */
@@ -5272,6 +5320,7 @@ main(int argc, char *argv[])
     ENGINE_NODE(bfd_chassis, "bfd_chassis");
     ENGINE_NODE(dns_cache, "dns_cache");
     ENGINE_NODE(route, "route");
+    ENGINE_NODE(route_exchange, "route_exchange");
 
 #define SB_NODE(NAME, NAME_STR) ENGINE_NODE_SB(NAME, NAME_STR);
     SB_NODES
@@ -5303,6 +5352,8 @@ main(int argc, char *argv[])
     engine_add_input(&en_route, &en_sb_advertised_route,
                      route_sb_advertised_route_data_handler);
 
+    engine_add_input(&en_route_exchange, &en_route, NULL);
+
     engine_add_input(&en_addr_sets, &en_sb_address_set,
                      addr_sets_sb_address_set_handler);
     engine_add_input(&en_port_groups, &en_sb_port_group,
@@ -5488,9 +5539,8 @@ main(int argc, char *argv[])
                      controller_output_mac_cache_handler);
     engine_add_input(&en_controller_output, &en_bfd_chassis,
                      controller_output_bfd_chassis_handler);
-    /* This is just temporary until the route output is actually used. */
-    engine_add_input(&en_controller_output, &en_route,
-                     controller_output_bfd_chassis_handler);
+    engine_add_input(&en_controller_output, &en_route_exchange,
+                     controller_output_route_exchange_handler);
 
     struct engine_arg engine_arg = {
         .sb_idl = ovnsb_idl_loop.idl,
@@ -6215,6 +6265,7 @@ main(int argc, char *argv[])
 
             poll_block();
         }
+        route_exchange_cleanup();
     }
 
     free(ovn_version);
@@ -6244,6 +6295,7 @@ main(int argc, char *argv[])
     service_stop();
     ovsrcu_exit();
     dns_resolve_destroy();
+    route_exchange_destroy();
 
     exit(retval);
 }

controller/route-exchange-stub.c

@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+
+#include <stdbool.h>
+
+#include "openvswitch/compiler.h"
+#include "route-exchange.h"
+
+bool
+route_exchange_relevant_port(const struct sbrec_port_binding *pb OVS_UNUSED)
+{
+    return false;
+}
+
+void
+route_exchange_run(struct route_exchange_ctx_in *r_ctx_in OVS_UNUSED,
+                   struct route_exchange_ctx_out *r_ctx_out OVS_UNUSED)
+{
+}
+
+void
+route_exchange_cleanup(void)
+{
+}
+
+void
+route_exchange_destroy(void)
+{
+}

controller/route-exchange.c

@@ -0,0 +1,102 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+
+#include <errno.h>
+#include <net/if.h>
+
+#include "openvswitch/vlog.h"
+
+#include "lib/ovn-sb-idl.h"
+
+#include "binding.h"
+#include "ha-chassis.h"
+#include "local_data.h"
+#include "route.h"
+#include "route-exchange.h"
+#include "route-exchange-netlink.h"
+
+
+VLOG_DEFINE_THIS_MODULE(route_exchange);
+static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
+
+static struct sset _maintained_vrfs = SSET_INITIALIZER(&_maintained_vrfs);
+
+void
+route_exchange_run(struct route_exchange_ctx_in *r_ctx_in,
+                   struct route_exchange_ctx_out *r_ctx_out OVS_UNUSED)
+{
+    struct sset old_maintained_vrfs = SSET_INITIALIZER(&old_maintained_vrfs);
+    sset_swap(&_maintained_vrfs, &old_maintained_vrfs);
+
+    const struct advertise_datapath_entry *ad;
+    HMAP_FOR_EACH (ad, node, r_ctx_in->announce_routes) {
+        struct hmap received_routes
+                = HMAP_INITIALIZER(&received_routes);
+        char vrf_name[IFNAMSIZ + 1];
+        snprintf(vrf_name, sizeof vrf_name, "ovnvrf%"PRIi64,
+                 ad->key);
+
+        if (ad->maintain_vrf) {
+            int error = re_nl_create_vrf(vrf_name, ad->key);
+            if (error && error != EEXIST) {
+                VLOG_WARN_RL(&rl,
+                             "Unable to create VRF %s for datapath "
+                             "%"PRId64": %s.",
+                             vrf_name, ad->key,
+                             ovs_strerror(error));
+                continue;
+            }
+            sset_add(&_maintained_vrfs, vrf_name);
+        } else {
+            /* a previous maintain-vrf flag was removed. We should therfor
+             * also not delete it even if we created it previously. */
+            sset_find_and_delete(&_maintained_vrfs, vrf_name);
+            sset_find_and_delete(&old_maintained_vrfs, vrf_name);
+        }
+
+        re_nl_sync_routes(ad->key, &ad->routes);
+    }
+
+    /* Remove VRFs previously maintained by us not found in the above loop. */
+    const char *vrf_name;
+    SSET_FOR_EACH_SAFE (vrf_name, &old_maintained_vrfs) {
+        if (!sset_find(&_maintained_vrfs, vrf_name)) {
+            re_nl_delete_vrf(vrf_name);
+        }
+        sset_delete(&old_maintained_vrfs, SSET_NODE_FROM_NAME(vrf_name));
+    }
+    sset_destroy(&old_maintained_vrfs);
+}
+
+void
+route_exchange_cleanup(void)
+{
+    const char *vrf_name;
+    SSET_FOR_EACH_SAFE (vrf_name, &_maintained_vrfs) {
+        re_nl_delete_vrf(vrf_name);
+    }
+}
+
+void
+route_exchange_destroy(void)
+{
+    const char *vrf_name;
+    SSET_FOR_EACH_SAFE (vrf_name, &_maintained_vrfs) {
+        sset_delete(&_maintained_vrfs, SSET_NODE_FROM_NAME(vrf_name));
+    }
+
+    sset_destroy(&_maintained_vrfs);
+}

controller/route-exchange.h

@@ -0,0 +1,33 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ROUTE_EXCHANGE_H
+#define ROUTE_EXCHANGE_H 1
+
+#include <stdbool.h>
+
+struct route_exchange_ctx_in {
+    /* Contains struct advertise_datapath_entry */
+    struct hmap *announce_routes;
+};
+
+struct route_exchange_ctx_out {
+};
+
+void route_exchange_run(struct route_exchange_ctx_in *,
+                        struct route_exchange_ctx_out *);
+void route_exchange_cleanup(void);
+void route_exchange_destroy(void);
+
+#endif /* ROUTE_EXCHANGE_H */

tests/ovs-macros.at

@@ -285,6 +285,17 @@ m4_define([OVS_WAIT_UNTIL],
   [check_ovs_wait_until_args "$#" "$2"
    OVS_WAIT([$1], [$2], [AT_LINE], [until $1])])
 
+dnl OVS_WAIT_UNTIL_EQUAL(COMMAND, OUTPUT)
+dnl
+dnl Executes shell COMMAND in a loop until it returns zero and the output
+dnl equals OUTPUT.  If COMMAND does not return zero or a desired output within
+dnl a reasonable time limit, fails the test.
+m4_define([OVS_WAIT_UNTIL_EQUAL],
+  [AT_FAIL_IF([test "$#" -ge 3])
+   echo "$2" > wait_until_expected
+   OVS_WAIT_UNTIL([$1 | diff -u wait_until_expected - ])])
+
+
 dnl OVS_WAIT_FOR_OUTPUT(COMMAND, EXIT-STATUS, STDOUT, STDERR)
 dnl OVS_WAIT_FOR_OUTPUT_UNQUOTED(COMMAND, EXIT-STATUS, STDOUT, STDERR)
 dnl

tests/system-common-macros.at

@@ -542,3 +542,18 @@ m4_define([CHECK_VRF],
     AT_SKIP_IF([test $rc -ne 0])
     on_exit 'modprobe -r vrf'
 ])
+
+# VRF_RESERVE([id])
+#
+# Helper to ensure we actually support vrfs and the vrf in question has no
+# route entries in it and is not existing.
+# We need to add it before deleting as routes can actually survive in a
+# deleted vrf.
+m4_define([VRF_RESERVE],
+    [
+     CHECK_VRF()
+     ip link add "ovnvrf$1" type vrf table "$1"
+     ip route flush vrf "ovnvrf$1"
+     ip link del "ovnvrf$1"
+    ]
+)