OpenYurt Raven 跨物理区域通信

$ kubectl get nodes -o wide
NAME          STATUS   ROLES                         AGE   VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                           KERNEL-VERSION                 CONTAINER-RUNTIME
edge-node-1   Ready    <none>                        23d   v1.22.3   10.0.0.210      <none>        Rocky Linux 8.6 (Green Obsidian)   4.18.0-372.13.1.el8_6.x86_64   containerd://1.6.24
edge-node-2   Ready    <none>                        17d   v1.22.3   10.0.0.80       <none>        Rocky Linux 8.6 (Green Obsidian)   4.18.0-372.13.1.el8_6.x86_64   containerd://1.6.24
yurt-cloud    Ready    control-plane,master,worker   24d   v1.22.3   172.20.163.65   <none>        Rocky Linux 8.9 (Green Obsidian)   4.18.0-477.27.1.el8_8.x86_64   containerd://1.6.4

$ kubectl get po -n kube-system -o wide | grep raven-agent-ds
raven-agent-ds-h27kh                 1/1     Running   0                16d   172.20.163.65   yurt-cloud    <none>           <none>
raven-agent-ds-js2lz                 1/1     Running   0                16d   10.0.0.80       edge-node-2   <none>           <none>
raven-agent-ds-wnsnc                 1/1     Running   0                16d   10.0.0.210      edge-node-1   <none>           <none>

$ kubectl get po -n kube-flannel
NAME                    READY   STATUS    RESTARTS   AGE
kube-flannel-ds-9ffgd   1/1     Running   0          24d
kube-flannel-ds-nw2ds   1/1     Running   0          23d
kube-flannel-ds-rp4hl   1/1     Running   0          17d

$ ethtool -i flannel.1
driver: vxlan
version: 0.1
firmware-version:
expansion-rom-version:
bus-info:
supports-statistics: no
supports-test: no
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: no

$ ping 10.0.0.80
PING 10.0.0.80 (10.0.0.80) 56(84) bytes of data.
^C
--- 10.0.0.80 ping statistics ---
14 packets transmitted, 0 received, 100% packet loss, time 13348ms

ping 10.0.0.210
PING 10.0.0.210 (10.0.0.210) 56(84) bytes of data.
^C
--- 10.0.0.210 ping statistics ---
5 packets transmitted, 0 received, 100% packet loss, time 4132ms

$ kubectl get po -o wide | grep nginx
nginx                                                 1/1     Running            0                15d    10.233.68.7    edge-node-1   <none>           <none>

$ curl -i http://10.233.68.7
HTTP/1.1 200 OK
Server: nginx/1.14.2
Date: Thu, 07 Dec 2023 09:43:43 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 04 Dec 2018 14:44:49 GMT
Connection: keep-alive
ETag: "5c0692e1-264"
Accept-Ranges: bytes

$ kubectl get gw gw-cloud -o jsonpath='{.status}' | jq
{
  "activeEndpoints": [
    {
      "config": {
        "enable-l3-tunnel": "true"
      },
      "nodeName": "yurt-cloud",
      "port": 4500,
      "publicIP": "172.20.163.65",
      "type": "tunnel"
    }
  ],
  "nodes": [
    {
      "nodeName": "yurt-cloud",
      "privateIP": "172.20.163.65",
      "subnets": [
        "10.233.64.0/24"
      ]
    }
  ]
}

$ tcpdump -nne -i any tcp and host 10.233.68.7 and port 80
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on any, link-type LINUX_SLL (Linux cooked v1), capture size 262144 bytes
10:33:15.740815  In 78:2c:29:fe:2a:46 ethertype IPv4 (0x0800), length 76: 10.233.68.7.80 > 10.233.64.0.47322: Flags [S.], seq 3602181374, ack 1858373815, win 27960, options [mss 1410,sackOK,TS val 3744027371 ecr 2615033249,nop,wscale 7], length 0
10:33:15.742060  In 78:2c:29:fe:2a:46 ethertype IPv4 (0x0800), length 68: 10.233.68.7.80 > 10.233.64.0.47322: Flags [.], ack 76, win 219, options [nop,nop,TS val 3744027373 ecr 2615033251], length 0
10:33:15.743899  In 78:2c:29:fe:2a:46 ethertype IPv4 (0x0800), length 306: 10.233.68.7.80 > 10.233.64.0.47322: Flags [P.], seq 1:239, ack 76, win 219, options [nop,nop,TS val 3744027375 ecr 2615033251], length 238: HTTP: HTTP/1.1 200 OK
10:33:15.744384  In 78:2c:29:fe:2a:46 ethertype IPv4 (0x0800), length 680: 10.233.68.7.80 > 10.233.64.0.47322: Flags [P.], seq 239:851, ack 76, win 219, options [nop,nop,TS val 3744027375 ecr 2615033251], length 612: HTTP
10:33:15.745554  In 78:2c:29:fe:2a:46 ethertype IPv4 (0x0800), length 68: 10.233.68.7.80 > 10.233.64.0.47322: Flags [F.], seq 851, ack 77, win 219, options [nop,nop,TS val 3744027376 ecr 2615033255], length 0

$ tcpdump -i eth0 -nv udp[46:4]=0x0ae94407

func Run(ctx context.Context, cfg *config.CompletedConfig) error {
    routeDriver, err := routedriver.New(cfg.RouteDriver, cfg.Config)
    if err != nil {
        return fmt.Errorf("fail to create route driver: %s, %s", cfg.RouteDriver, err)
    }
    err = routeDriver.Init()
    if err != nil {
        return fmt.Errorf("fail to initialize route driver: %s, %s", cfg.RouteDriver, err)
    }
    klog.Infof("route driver %s initialized", cfg.RouteDriver)
    vpnDriver, err := vpndriver.New(cfg.VPNDriver, cfg.Config)
    if err != nil {
        return fmt.Errorf("fail to create vpn driver: %s, %s", cfg.VPNDriver, err)
    }
    err = vpnDriver.Init()
    if err != nil {
        return fmt.Errorf("fail to initialize vpn driver: %s, %s", cfg.VPNDriver, err)
    }
    klog.Infof("VPN driver %s initialized", cfg.VPNDriver)
    // a lot of code here
}

func NewEngineController(nodeName string, forwardNodeIP bool, routeDriver routedriver.Driver, manager manager.Manager,
    vpnDriver vpndriver.Driver) (*EngineController, error) {
    ctr := &EngineController{
        nodeName:      nodeName,
        forwardNodeIP: forwardNodeIP,
        queue:         workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
        routeDriver:   routeDriver,
        manager:       manager,
        vpnDriver:     vpnDriver,
    }

    err := ctrl.NewControllerManagedBy(ctr.manager).
        For(&v1alpha1.Gateway{}, builder.WithPredicates(predicate.Funcs{
            CreateFunc: ctr.addGateway,
            UpdateFunc: ctr.updateGateway,
            DeleteFunc: ctr.deleteGateway,
        })).
        Complete(reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
            return reconcile.Result{}, nil
        }))
    if err != nil {
        klog.ErrorS(err, "failed to new raven agent controller with manager")
    }
    ctr.ravenClient = ctr.manager.GetClient()

    return ctr, nil
}

func (c *EngineController) sync() error {
    // a lot of code here
    klog.InfoS("applying network", "localEndpoint", nw.LocalEndpoint, "remoteEndpoint", nw.RemoteEndpoints)
    err = c.vpnDriver.Apply(nw, c.routeDriver.MTU)
    if err != nil {
        return err
    }
    err = c.routeDriver.Apply(nw, c.vpnDriver.MTU)
    if err != nil {
        return err
    }
    // ...
}

func (l *libreswan) Apply(network *types.Network, routeDriverMTUFn func(*types.Network) (int, error)) (err error) {

    // a lot of code here
    for name, connection := range desiredConnections {
        err := l.connectToEndpoint(name, connection)
        errList = errList.Append(err)
    }
}

func (l *libreswan) connectToEndpoint(name string, connection *vpndriver.Connection) errorlist.List {
    errList := errorlist.List{}
    if _, ok := l.connections[name]; ok {
        klog.InfoS("skipping connect because connection already exists", "connectionName", name)
        return errList
    }
    err := l.whackConnectToEndpoint(name, connection)
    if err != nil {
        errList = errList.Append(err)
        klog.ErrorS(err, "error connect connection", "connectionName", name)
        return errList
    }
    l.connections[name] = connection
    return errList
}

func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error)) (err error) {
    if network.LocalEndpoint == nil || len(network.RemoteEndpoints) == 0 {
        klog.Info("no local gateway or remote gateway is found, cleaning up route setting")
        return vx.Cleanup()
    }
    if len(network.LocalNodeInfo) == 1 {
        klog.Infof("only gateway node exist in current gateway, cleaning up route setting")
        return vx.Cleanup()
    }
    // a lot of code here
}