Initial commit

.dockerignore

@@ -0,0 +1 @@
+/plugin/

.gitignore

@@ -0,0 +1,4 @@
+/plugin/
+__pycache__/
+
+.vscode/

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.7-alpine
+
+COPY requirements.txt /opt/
+RUN apk --no-cache add gcc musl-dev && \
+    pip install -r /opt/requirements.txt && \
+    apk --no-cache del gcc musl-dev
+
+RUN mkdir -p /opt/plugin /run/docker/plugins /var/run/docker/netns
+COPY net-dhcp/ /opt/plugin/net_dhcp
+
+WORKDIR /opt/plugin
+ENTRYPOINT ["python", "-m", "net_dhcp"]

Makefile

@@ -0,0 +1,40 @@
+PLUGIN_NAME = registry.dierkse.nl/net-dhcp
+PLUGIN_TAG ?= latest
+
+all: clean build rootfs create enable
+
+clean:
+	@echo "### rm ./plugin"
+	@rm -rf ./plugin
+
+build:
+	@echo "### docker build: rootfs image with net-dhcp"
+	@docker build -t ${PLUGIN_NAME}:rootfs .
+
+rootfs:
+	@echo "### create rootfs directory in ./plugin/rootfs"
+	@mkdir -p ./plugin/rootfs
+	@docker create --name tmp ${PLUGIN_NAME}:rootfs
+	@docker export tmp | tar -x -C ./plugin/rootfs
+	@echo "### copy config.json to ./plugin/"
+	@cp config.json ./plugin/
+	@docker rm -vf tmp
+
+create:
+	@echo "### remove existing plugin ${PLUGIN_NAME}:${PLUGIN_TAG} if exists"
+	@docker plugin rm -f ${PLUGIN_NAME}:${PLUGIN_TAG} || true
+	@echo "### create new plugin ${PLUGIN_NAME}:${PLUGIN_TAG} from ./plugin"
+	@docker plugin create ${PLUGIN_NAME}:${PLUGIN_TAG} ./plugin
+
+debug:
+	@docker run --rm -ti --cap-add CAP_SYS_ADMIN --network host --volume /run/docker/plugins:/run/docker/plugins \
+		--volume /run/docker.sock:/run/docker.sock --volume /var/run/docker/netns:/var/run/docker/netns \
+		${PLUGIN_NAME}:rootfs
+
+enable:
+	@echo "### enable plugin ${PLUGIN_NAME}:${PLUGIN_TAG}"		
+	@docker plugin enable ${PLUGIN_NAME}:${PLUGIN_TAG}
+
+push:
+	@echo "### push plugin ${PLUGIN_NAME}:${PLUGIN_TAG}"
+	@docker plugin push ${PLUGIN_NAME}:${PLUGIN_TAG}

README.md

@@ -0,0 +1,103 @@
+# docker-net-dhcp
+`docker-net-dhcp` is a Docker plugin providing a network driver which allocates IP addresses (IPv4 and optionally IPv6)
+via an existing DHCP server (e.g. your router).
+
+When configured correctly, this allows you to spin up a container (e.g. `docker run ...` or `docker-compose up ...`) and
+access it on your network as if it was any other machine!
+
+# Usage
+## Installation
+```
+$ docker plugin install devplayer0/net-dhcp
+Plugin "devplayer0/net-dhcp" is requesting the following privileges:
+ - network: [host]
+ - host pid namespace: [true]
+ - mount: [/var/run/docker.sock]
+ - capabilities: [CAP_NET_ADMIN CAP_SYS_ADMIN]
+Do you grant the above permissions? [y/N] y
+latest: Pulling from devplayer0/net-dhcp
+<some id>: Download complete 
+Digest: sha256:<some hash>
+Status: Downloaded newer image for devplayer0/net-dhcp:latest
+Installed plugin devplayer0/net-dhcp
+$
+```
+
+## Network creation
+In order to create a Docker network using `net-dhcp`, you'll need a pre-configured bridge interface on the host. How you
+set this up will depend on your system, but the following (manual) instructions should work on most Linux distros:
+```
+# Create the bridge
+$ sudo ip link add my-bridge type bridge
+$ sudo ip link set my-bridge up
+
+# Assuming 'eth0' is connected to your LAN (where the DHCP server is)
+$ sudo ip link set eth0 up
+# Attach your network card to the bridge
+$ sudo ip link set eth0 master my-bridge
+
+# Get an IP for the host (will go out to the DHCP server since eth0 is attached to the bridge)
+$ sudo dhcpcd my-bridge
+```
+
+Once the bridge is ready, you can create the network:
+```
+$ docker network create -d devplayer0/net-dhcp:latest --ipam-driver null -o bridge=my-bridge my-dhcp-net
+<some network id>
+$
+
+# With IPv6 enabled
+# Although `docker network create` has a `--ipv6` flag, it doesn't work with the null IPAM driver
+$ docker network create -d devplayer0/net-dhcp:latest --ipam-driver null -o bridge=test -o ipv6=true my-dhcp-net
+<some network id>
+$
+```
+_Note: The `null` IPAM driver **must** be used, or else Docker will try to allocate IP addresses from its choice of
+subnet - this can cause IP conflicts since the bridge is connected to your local network!_
+
+## Container creation
+Once you've created a network, you can create some containers:
+```
+$ docker run --rm -ti --network my-dhcp-net alpine
+/ # ip address show
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1000
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
+    inet 127.0.0.1/8 scope host lo
+       valid_lft forever preferred_lft forever
+159: my-bridge0@if160: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue state UP qlen 1000
+    link/ether 86:41:68:f8:85:b9 brd ff:ff:ff:ff:ff:ff
+    inet 10.255.0.246/24 brd 10.255.0.255 scope global test0
+       valid_lft forever preferred_lft forever
+/ # ip route show
+default via 10.255.0.123 dev my-bridge0 
+10.255.0.0/24 dev my-bridge0 scope link  src 10.255.0.246 
+/ #
+```
+Note:
+ - It will take a bit longer than usual for the container to start, as a DHCP lease needs to be obtained before creating it
+ - Once created, a persistent DHCP client will renew the DHCP lease (and then update the default gateway in the
+ container) when necessary - **this client runs separately from the container**
+ - Use `--mac-address` to specify a MAC address if you've configured reserved IP addresses on your DHCP server, or if
+ you want a container to re-use an old lease
+
+# Implementation
+Fundamentally, the same mechanism is used by `net-dhcp` as Docker's `bridge` driver to wire up networking to containers.
+That is, a bridge on the host is used as a switch so that containers can communicate with each other - `veth` pairs
+connect each container's network namespace to the bridge.
+
+- While Docker creates and manages its own bridges (and routes and filters traffic), `net-dhcp` uses an existing bridge
+on the host, bridged with the desired local network. 
+- Instead of allocating IP addresses from a static pool stored on the Docker host, `net-dhcp` relies on an external DHCP
+server to provide IP addresses
+
+## Flow
+1. Container creation request is made
+2. A `veth` pair is created and the host end is connected to the bridge (at this point both interfaces are still in the
+host namespace)
+3. A DHCP client (BusyBox `udhcpc`) is started on the container end (still in the host namespace) - initial IP address
+is provided to Docker by the plugin
+4. Docker moves the container end of the `veth` pair into the container's network namespace and sets the IP address - at
+this point `udhcpc` must be stopped
+5. `net-dhcp` starts `udhcpc` on the container end of the `veth` pair in the container's **network namespace** (but
+still in the host / plugin **PID namespace** - this means that the container can't see the DHCP client)
+6. `udhcpc` continues to run, renewing the lease when required, until the container shuts down

config.json

@@ -0,0 +1,33 @@
+{
+    "description": "Docker host bridge DHCP networking",
+    "interface": {
+        "socket": "net-dhcp.sock",
+        "types": [
+            "docker.networkdriver/1.0"
+        ]
+    },
+    "entrypoint": [ "python", "-m", "net_dhcp" ],
+    "workdir": "/opt/plugin",
+    "network": {
+        "type": "host"
+    },
+    "mounts": [
+        {
+            "source": "/var/run/docker.sock",
+            "destination": "/run/docker.sock",
+            "type": "bind",
+            "options": [
+                "bind"
+            ]
+        }
+    ],
+    "pidhost": true,
+    "linux": {
+        "capabilities": [
+            "CAP_NET_ADMIN",
+            "CAP_SYS_ADMIN",
+            "CAP_SYS_PTRACE",
+            "CAP_SYS_RESOURCE"
+        ]
+    }
+}

net-dhcp/__init__.py

@@ -0,0 +1,23 @@
+import logging
+
+from flask import Flask, jsonify
+
+class NetDhcpError(Exception):
+    def __init__(self, status, *args):
+        Exception.__init__(self, *args)
+        self.status = status
+
+app = Flask(__name__)
+
+from . import network
+
+logger = logging.getLogger('gunicorn.error')
+
+@app.errorhandler(404)
+def err_not_found(_e):
+    return jsonify({'Err': 'API not found'}), 404
+
+@app.errorhandler(Exception)
+def err(e):
+    logger.exception(e)
+    return jsonify({'Err': str(e)}), 500

net-dhcp/__main__.py

@@ -0,0 +1,14 @@
+import logging
+import socketserver
+from werkzeug.serving import run_simple
+from . import app
+
+fh = logging.FileHandler('/var/log/net-dhcp.log')
+fh.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s'))
+
+logger = logging.getLogger('net-dhcp')
+logger.setLevel(logging.DEBUG)
+logger.addHandler(fh)
+
+socketserver.TCPServer.allow_reuse_address = True
+run_simple('unix:///run/docker/plugins/net-dhcp.sock', 0, app)

net-dhcp/interface.py

@@ -0,0 +1,203 @@
+import atexit
+import pyroute2
+
+#-------------------------------
+
+def InitializeLibrary(library):
+    if library == 'NDB':
+        global ndb
+        ndb = pyroute2.NDB()
+        @atexit.register
+        def close_ndb():
+            ndb.close()
+    elif library == 'IPR':
+        global iproute
+        iproute = pyroute2.IPRoute()
+        @atexit.register
+        def close_iproute():
+            iproute.close()
+
+#-------------------------------
+
+def GetInterfaces(library):
+    interfaces = []
+    if library == 'NDB':
+        for itf in ndb.interfaces:
+            interfaces.append(Interface(ndb.interfaces[itf.ifname]))
+    elif library == 'IPR':
+        for itf in iproute.get_links():
+            interfaces.append(Interface(itf))
+    return interfaces
+
+def GetInterface(library, ifname):
+    if library == 'NDB':
+        return Interface(ndb.interfaces[ifname])
+    elif library == 'IPR':
+        return Interface(iproute.get_links(ifname=ifname)[0])
+    return None
+
+class Interface:
+    def __init__(self, interface):
+        self.ndb = None
+        self.ipr = None
+
+        self._set_interface_(interface)
+        self._initialize_attributes_()
+
+    def __getitem__(self, key):
+        return getattr(self, key, None)
+
+    def _set_interface_(self, interface):
+        if isinstance(interface, pyroute2.netlink.rtnl.ifinfmsg.ifinfmsg):
+            self.ipr = interface
+        elif isinstance(interface, pyroute2.ndb.objects.interface.Interface):
+            self.ndb = interface
+
+    def _initialize_attributes_(self):
+        if self.ndb:
+            self.index = self.ndb['index']
+            self.ifname = self.ndb['ifname']
+            self.kind = self.ndb['kind']
+            self.address = self.ndb['address']
+            if 'netns' in self.ndb:
+                 self.netns = self.ndb['netns']
+            else:
+                 self.netns = None
+            self.ipaddr = []
+            for address in self.ndb.ipaddr:
+                ipaddress = {}
+                ipaddress['address'] = address['address']
+                ipaddress['prefixlen'] = address['prefixlen']
+                self.ipaddr.append(ipaddress)
+            self.routes = []
+            for r in self.ndb.routes:
+                route = {}
+                route['type'] = r['type']
+                route['family'] = r['family']
+                route['dst'] = r['dst']
+                route['gateway'] = r['gateway']
+                route['dst_len'] = r['dst_len']
+                self.routes.append(route)
+        elif self.ipr:
+            self.index = self.ipr['index']
+            self.ifname = self.ipr.get_attr('IFLA_IFNAME')
+            self.kind = self.ipr.get_attr('IFLA_LINKINFO').get_attr('IFLA_INFO_KIND') if self.ipr.get_attr('IFLA_LINKINFO') else None
+            self.address = self.ipr.get_attr('IFLA_ADDRESS')
+            self.netns = None
+            self.ipaddr = []
+            for address in iproute.get_addr(label=self.ipr.get_attr('IFLA_IFNAME')):
+                ipaddress = {}
+                ipaddress['address'] = address.get_attr('IFA_ADDRESS')
+                ipaddress['prefixlen'] = address['prefixlen']
+                self.ipaddr.append(ipaddress)
+            self.routes = []
+            for r in iproute.get_routes():
+                attrs = dict(r['attrs'])
+                if attrs['RTA_OIF'] == self['index']:
+                    if 'RTA_PRIORITY' in attrs and not attrs['RTA_PRIORITY'] < 255:
+                        continue
+                    route = {}
+                    route['type'] = r['type']
+                    route['family'] = r['family']
+                    route['dst'] = attrs['RTA_DST'] if 'RTA_DST' in attrs else ''
+                    route['gateway'] = attrs['RTA_GATEWAY'] if 'RTA_GATEWAY' in attrs else None
+                    route['dst_len'] = r['dst_len']
+                    self.routes.append(route)
+
+    def up(self):
+        if self.ndb:
+             self.ndb.set('state', 'up').commit()
+        elif self.ipr:
+             iproute.link("set", index=self.index, state="up")
+
+    def down(self):
+        if self.ndb:
+             self.ndb.set('state', 'down').commit()
+        elif self.ipr:
+             iproute.link("set", index=self.index, state="down")
+
+def CreateInterface(library, ifname, kind, peer):
+    try:
+        if library == 'NDB':
+            ndb.interfaces.create(ifname=ifname, kind=kind, peer=peer).commit()
+            return Interface(ndb.interfaces[ifname])
+        elif library == 'IPR':
+            iproute.link("add", ifname=ifname, kind=kind, peer=peer)
+            return Interface(iproute.get_links(ifname=ifname)[0])
+        return None
+    except Exception as e:
+        return None
+
+def RemoveInterface(library, ifname):
+    try:
+        if library == 'NDB':
+            ndb.interfaces[ifname].remove().commit()
+        elif library == 'IPR':
+            iproute.link("del", index=iproute.link_lookup(ifname=ifname)[0])
+    except Exception as e:
+        return
+
+def AddPort(library, bridge, ifname):
+    try:
+        if library == 'NDB':
+            ndb.interfaces[bridge].add_port(ifname).commit()
+        elif library == 'IPR':
+            iproute.link("set", index=iproute.link_lookup(ifname=ifname)[0], master=iproute.link_lookup(ifname=bridge)[0])
+    except Exception as e:
+        return
+
+def DelPort(library, bridge, ifname):
+    try:
+        if library == 'NDB':
+            ndb.interfaces[bridge].del_port(ifname).commit()
+        elif library == 'IPR':
+            iproute.link("set", index=iproute.link_lookup(ifname=ifname)[0], master=0)
+    except Exception as e:
+        return
+
+#-------------------------------
+
+def test_Variable(v1, v2, f):
+    assert v1 == v2, "{} not equal ({}, {})".format(f, v1, v2)
+
+def test_Interface():
+    itf1 = GetInterface('NDB', 'br0')
+    itf2 = GetInterface('IPR', 'br0')
+
+    test_Variable(itf1.ifname, itf2.ifname, 'ifname')
+    test_Variable(itf1.kind, itf2.kind, 'kind')
+    test_Variable(itf1.ipaddr(), itf2.ipaddr(), 'ipaddr')
+    test_Variable(itf1.routes, itf2.routes, 'routes')
+
+    print("OK")
+#test_Interface()
+
+def test_CreateInterface():
+    CreateInterface('NDB', 'ndb0', 'veth', 'ndb1')
+    CreateInterface('IPR', 'ipr0', 'veth', 'ipr1')
+
+    itf1 = GetInterface('NDB', 'ndb0')
+    itf2 = GetInterface('IPR', 'ipr0')
+
+    itf1.up();
+    itf2.up();
+
+    itf3 = GetInterface('NDB', 'ndb1')
+    itf4 = GetInterface('IPR', 'ipr1')
+
+    itf3.up();
+    itf4.up();
+
+    AddPort('NDB', 'br0', 'ndb0')
+    AddPort('IPR', 'br0', 'ipr0')
+    
+#test_CreateInterface()
+
+def test_RemoveInterface():
+    DelPort('NDB', 'br0', 'ndb0')
+    DelPort('IPR', 'br0', 'ipr0')
+
+    RemoveInterface('NDB', 'ndb0')
+    RemoveInterface('IPR', 'ipr0')
+#test_RemoveInterface()
+

net-dhcp/network.py

@@ -0,0 +1,388 @@
+import itertools
+import ipaddress
+import logging
+import atexit
+import socket
+import time
+import threading
+import subprocess
+
+import pyroute2
+from pyroute2.netlink.rtnl import rtypes
+import docker
+from flask import request, jsonify
+
+from . import interface
+from . import NetDhcpError, udhcpc, app
+
+LIBRARY = 'IPR'
+
+OPTS_KEY = 'com.docker.network.generic'
+OPT_BRIDGE = 'bridge'
+OPT_IPV6 = 'ipv6'
+
+logger = logging.getLogger('net-dhcp')
+
+interface.InitializeLibrary(LIBRARY)
+
+client = docker.from_env()
+@atexit.register
+def close_docker():
+    client.close()
+
+gateway_hints = {}
+container_dhcp_clients = {}
+@atexit.register
+def cleanup_dhcp():
+    for endpoint, dhcp in container_dhcp_clients.items():
+        logger.warning('cleaning up orphaned container DHCP client (endpoint "%s")', endpoint)
+        dhcp.stop()
+
+def veth_pair(e):
+    return f'dh-{e[:12]}', f'{e[:12]}-dh'
+
+def iface_addrs(iface):
+    return list(map(lambda a: ipaddress.ip_interface((a['address'], a['prefixlen'])), iface.ipaddr))
+def iface_nets(iface):
+    return list(map(lambda n: n.network, iface_addrs(iface)))
+
+def get_bridges():
+    reserved_nets = set(map(ipaddress.ip_network, map(lambda c: c['Subnet'], \
+        itertools.chain.from_iterable(map(lambda i: i['Config'], filter(lambda i: i['Driver'] != 'net-dhcp', \
+            map(lambda n: n.attrs['IPAM'], client.networks.list())))))))
+
+    return dict(map(lambda i: (i['ifname'], i), filter(lambda i: i['kind'] == 'bridge' and not \
+        set(iface_nets(i)).intersection(reserved_nets), map(lambda i: interface.GetInterface(LIBRARY, i['ifname']), \
+            interface.GetInterfaces(LIBRARY)))))
+
+def net_bridge(n):
+    return interface.GetInterface(LIBRARY, client.networks.get(n).attrs['Options'][OPT_BRIDGE])
+def ipv6_enabled(n):
+    options = client.networks.get(n).attrs['Options']
+    return OPT_IPV6 in options and options[OPT_IPV6] == 'true'
+
+def endpoint_container_iface(n, e):
+    for cid, info in client.networks.get(n).attrs['Containers'].items():
+        if info['EndpointID'] == e:
+            container = client.containers.get(cid)
+            netns = f'/proc/{container.attrs["State"]["Pid"]}/ns/net'
+
+            with pyroute2.NetNS(netns) as rtnl:
+                for link in rtnl.get_links():
+                    attrs = dict(link['attrs'])
+                    if attrs['IFLA_ADDRESS'] == info['MacAddress']:
+                        return {
+                            'netns': netns,
+                            'ifname': attrs['IFLA_IFNAME'],
+                            'address': attrs['IFLA_ADDRESS']
+                        }
+            break
+    return None
+def await_endpoint_container_iface(n, e, timeout=5):
+    start = time.time()
+    iface = None
+    while time.time() - start < timeout:
+        try:
+            iface = endpoint_container_iface(n, e)
+        except docker.errors.NotFound:
+            time.sleep(0.5)
+    if not iface:
+        raise NetDhcpError('Timed out waiting for container to become availabile')
+    return iface
+
+def endpoint_container_hostname(n, e):
+    for cid, info in client.networks.get(n).attrs['Containers'].items():
+        if info['EndpointID'] == e:
+            return client.containers.get(cid).attrs['Config']['Hostname']
+    return None
+
+@app.route('/NetworkDriver.GetCapabilities', methods=['POST'])
+def net_get_capabilities():
+    return jsonify({
+        'Scope': 'local',
+        'ConnectivityScope': 'global'
+    })
+
+@app.route('/NetworkDriver.CreateNetwork', methods=['POST'])
+def create_net():
+    req = request.get_json(force=True)
+    for data in req['IPv4Data']:
+        if data['AddressSpace'] != 'null' or data['Pool'] != '0.0.0.0/0':
+            return jsonify({'Err': 'Only the null IPAM driver is supported'}), 400
+
+    options = req['Options'][OPTS_KEY]
+    if OPT_BRIDGE not in options:
+        return jsonify({'Err': 'No bridge provided'}), 400
+    # We have to use a custom "enable IPv6" option because Docker's null IPAM driver doesn't support IPv6 and a plugin
+    # IPAM driver isn't allowed to return an empty address
+    if OPT_IPV6 in options and options[OPT_IPV6] not in ('', 'true', 'false'):
+        return jsonify({'Err': 'Invalid boolean value for ipv6'}), 400
+
+    desired = options[OPT_BRIDGE]
+    bridges = get_bridges()
+    if desired not in bridges:
+        return jsonify({'Err': f'Bridge "{desired}" not found (or the specified bridge is already used by Docker)'}), 400
+
+    logger.info('Creating network "%s" (using bridge "%s")', req['NetworkID'], desired)
+    return jsonify({})
+
+@app.route('/NetworkDriver.DeleteNetwork', methods=['POST'])
+def delete_net():
+    return jsonify({})
+
+@app.route('/NetworkDriver.CreateEndpoint', methods=['POST'])
+def create_endpoint():
+    req = request.get_json(force=True)
+    network_id = req['NetworkID']
+    endpoint_id = req['EndpointID']
+    req_iface = req['Interface']
+
+    bridge = net_bridge(network_id)
+    bridge_addrs = iface_addrs(bridge)
+
+    if_host, if_container = veth_pair(endpoint_id)
+    logger.info('creating veth pair %s <=> %s', if_host, if_container)
+    if_host = interface.CreateInterface(LIBRARY, if_host, 'veth', if_container)
+    if_host.up()
+
+    try:
+        start = time.time()
+        while isinstance(if_container, str) and time.time() - start < 10:
+            try:
+                if_container = interface.GetInterface(LIBRARY, if_container)
+                if_container.up()
+            except KeyError:
+                time.sleep(0.5)
+        if isinstance(if_container, str):
+            raise NetDhcpError(f'timed out waiting for {if_container} to appear in host')
+
+        interface.AddPort(LIBRARY, bridge.ifname, if_host.ifname)
+
+        res_iface = {
+            'MacAddress': '',
+            'Address': '',
+            'AddressIPv6': ''
+        }
+
+# !!!
+#        if 'MacAddress' in req_iface and req_iface['MacAddress']:
+#            (if_container
+#            .set('address', req_iface['MacAddress'])
+#            .commit())
+#        else:
+#            res_iface['MacAddress'] = if_container['address']
+# !!!
+        res_iface['MacAddress'] = if_container['address']
+
+        def try_addr(type_):
+            addr = None
+            k = 'AddressIPv6' if type_ == 'v6' else 'Address'
+            if k in req_iface and req_iface[k]:
+                # TODO: Should we allow static IP's somehow?
+                # Just validate the address, Docker will add it to the interface for us
+                #addr = ipaddress.ip_interface(req_iface[k])
+                #for bridge_addr in bridge_addrs:
+                #    if addr.ip == bridge_addr.ip:
+                #        raise NetDhcpError(400, f'Address {addr} is already in use on bridge {bridge["ifname"]}')
+                raise NetDhcpError('Only the null IPAM driver is supported')
+            else:
+                dhcp = udhcpc.DHCPClient(if_container, v6=type_ == 'v6', once=True)
+                addr = dhcp.finish()
+                if not addr:
+                    return
+                res_iface[k] = str(addr)
+
+                if dhcp.gateway:
+                    gateway_hints[endpoint_id] = dhcp.gateway
+            logger.info('Adding IP%s address %s to %s', type_, addr, if_container['ifname'])
+
+        try_addr('v4')
+        if ipv6_enabled(network_id):
+            try_addr('v6')
+
+        res = jsonify({
+            'Interface': res_iface
+        })
+    except Exception as e:
+        logger.exception(e)
+
+        if not isinstance(if_container, str):
+            interface.DelPort(LIBRARY, bridge.ifname, if_host.ifname)
+        interface.RemoveInterface(LIBRARY, if_host.ifname)
+
+        if isinstance(e, NetDhcpError):
+            res = jsonify({'Err': str(e)}), e.status
+        else:
+            res = jsonify({'Err': str(e)}), 500
+    finally:
+        return res
+
+@app.route('/NetworkDriver.EndpointOperInfo', methods=['POST'])
+def endpoint_info():
+    req = request.get_json(force=True)
+
+    bridge = net_bridge(req['NetworkID'])
+    if_host, _if_container = veth_pair(req['EndpointID'])
+    if_host = interface.GetInterface(LIBRARY, if_host)
+
+    return jsonify({
+        'bridge': bridge['ifname'],
+        'if_host': {
+            'name': if_host['ifname'],
+            'mac': if_host['address']
+        }
+    })
+
+@app.route('/NetworkDriver.DeleteEndpoint', methods=['POST'])
+def delete_endpoint():
+    req = request.get_json(force=True)
+
+    bridge = net_bridge(req['NetworkID'])
+    if_host, _if_container = veth_pair(req['EndpointID'])
+    if_host = interface.GetInterface(LIBRARY, if_host)
+
+    interface.DelPort(LIBRARY, bridge.ifname, if_host.ifname)
+    interface.RemoveInterface(LIBRARY, if_host.ifname)
+
+    return jsonify({})
+
+@app.route('/NetworkDriver.Join', methods=['POST'])
+def join():
+    req = request.get_json(force=True)
+    network = req['NetworkID']
+    endpoint = req['EndpointID']
+
+    bridge = net_bridge(req['NetworkID'])
+    _if_host, if_container = veth_pair(req['EndpointID'])
+
+    res = {
+        'InterfaceName': {
+            'SrcName': if_container,
+            'DstPrefix': bridge['ifname']
+        },
+        'StaticRoutes': []
+    }
+
+    if endpoint in gateway_hints:
+        gateway = gateway_hints[endpoint]
+        logger.info('Setting IPv4 gateway from DHCP (%s)', gateway)
+        res['Gateway'] = str(gateway)
+        del gateway_hints[endpoint]
+
+    ipv6 = ipv6_enabled(network)
+    for route in bridge.routes:
+        if route['type'] != rtypes['RTN_UNICAST'] or \
+            (route['family'] == socket.AF_INET6 and not ipv6):
+            continue
+
+        if route['dst'] in ('', '/0'):
+            if route['family'] == socket.AF_INET and 'Gateway' not in res:
+                logger.info('Adding IPv4 gateway %s', route['gateway'])
+                res['Gateway'] = route['gateway']
+            elif route['family'] == socket.AF_INET6 and 'GatewayIPv6' not in res:
+                logger.info('Adding IPv6 gateway %s', route['gateway'])
+                res['GatewayIPv6'] = route['gateway']
+        elif route['gateway']:
+            dst = f'{route["dst"]}/{route["dst_len"]}'
+            logger.info('Adding route to %s via %s', dst, route['gateway'])
+            res['StaticRoutes'].append({
+                'Destination': dst,
+                'RouteType': 0,
+                'NextHop': route['gateway']
+            })
+
+    container_dhcp_clients[endpoint] = ContainerDHCPManager(network, endpoint)
+    return jsonify(res)
+
+@app.route('/NetworkDriver.Leave', methods=['POST'])
+def leave():
+    req = request.get_json(force=True)
+    endpoint = req['EndpointID']
+
+    if endpoint in container_dhcp_clients:
+        container_dhcp_clients[endpoint].stop()
+        del container_dhcp_clients[endpoint]
+
+    return jsonify({})
+
+# Trying to grab the container's attributes (to get the network namespace)
+# will deadlock (since Docker is waiting on us), so we must defer starting
+# the DHCP client
+class ContainerDHCPManager:
+    def __init__(self, network, endpoint):
+        self.network = network
+        self.endpoint = endpoint
+        self.ipv6 = ipv6_enabled(network)
+
+        self.dhcp = None
+        self.dhcp6 = None
+        self._thread = threading.Thread(target=self.run)
+        self._thread.start()
+
+    def _on_event(self, dhcp, event_type, _event):
+        if event_type != udhcpc.EventType.RENEW or not dhcp.gateway:
+            return
+
+        logger.info('[dhcp container] Replacing gateway with %s', dhcp.gateway)
+        subprocess.check_call(['nsenter', f'-n{dhcp.netns}', '--', '/sbin/ip', 'route', 'replace', 'default', 'via',
+            str(dhcp.gateway)], timeout=1, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL)
+
+        # TODO: Adding default route with NDB seems to be broken (because of the dst syntax?)
+        #for route in ndb.routes:
+        #    if route['type'] != rtypes['RTN_UNICAST'] or \
+        #        route['oif'] != dhcp.iface['index'] or \
+        #        (route['family'] == socket.AF_INET6 and not self.ipv6) or \
+        #        route['dst'] not in ('', '/0'):
+        #        continue
+
+        #    # Needed because Route.remove() doesn't like a blank destination
+        #    logger.info('Removing default route via %s', route['gateway'])
+        #    route['dst'] = '::' if route['family'] == socket.AF_INET6 else '0.0.0.0'
+        #    (route
+        #        .remove()
+        #        .commit())
+
+        #logger.info('Adding default route via %s', dhcp.gateway)
+        #(ndb.routes.add({'oif': dhcp.iface['index'], 'gateway': dhcp.gateway})
+        #    .commit())
+
+    def run(self):
+        try:
+            iface = await_endpoint_container_iface(self.network, self.endpoint)
+            hostname = endpoint_container_hostname(self.network, self.endpoint)
+
+            self.dhcp = udhcpc.DHCPClient(iface, event_listener=self._on_event, hostname=hostname)
+            logger.info('Starting DHCPv4 client on %s in container namespace %s', iface['ifname'], \
+                self.dhcp.netns)
+
+            if self.ipv6:
+                self.dhcp6 = udhcpc.DHCPClient(iface, v6=True, event_listener=self._on_event, hostname=hostname)
+                logger.info('Starting DHCPv6 client on %s in container namespace %s', iface['ifname'], \
+                    self.dhcp6.netns)
+        except Exception as e:
+            logger.exception(e)
+            if self.dhcp:
+                self.dhcp.finish(timeout=1)
+
+    def stop(self):
+        if not self.dhcp:
+            return
+
+        try:
+            logger.info('Shutting down DHCPv4 client on %s in container namespace %s', \
+                self.dhcp.iface['ifname'], self.dhcp.netns)
+            self.dhcp.finish(timeout=1)
+        finally:
+            try:
+                if self.ipv6:
+                    logger.info('Shutting down DHCPv6 client on %s in container namespace %s', \
+                        self.dhcp6.iface['ifname'], self.dhcp.netns)
+                    self.dhcp6.finish(timeout=1)
+            finally:
+                self._thread.join()
+
+                # we have to do this since the docker client leaks sockets...
+                global client
+                client.close()
+                client = docker.from_env()

net-dhcp/udhcpc.py

@@ -0,0 +1,148 @@
+from enum import Enum
+import ipaddress
+import json
+import struct
+import binascii
+import os
+from os import path
+from select import select
+import threading
+import subprocess
+import logging
+
+from eventfd import EventFD
+import posix_ipc
+
+HANDLER_SCRIPT = path.join(path.dirname(__file__), 'udhcpc_handler.py')
+AWAIT_INTERVAL = 0.1
+VENDOR_ID = 'docker'
+
+class EventType(Enum):
+    BOUND = 'bound'
+    RENEW = 'renew'
+    DECONFIG = 'deconfig'
+    LEASEFAIL = 'leasefail'
+
+logger = logging.getLogger('net-dhcp')
+
+class DHCPClientError(Exception):
+    pass
+
+def _nspopen_wrapper(netns):
+    return lambda cmd, *args, **kwargs: subprocess.Popen(['nsenter', f'-n{netns}', '--'] + cmd, *args, **kwargs)
+class DHCPClient:
+    def __init__(self, iface, v6=False, once=False, hostname=None, event_listener=None):
+        self.iface = iface
+        self.v6 = v6
+        self.once = once
+        self.event_listeners = [DHCPClient._attr_listener]
+        if event_listener:
+            self.event_listeners.append(event_listener)
+
+        self.netns = None
+        if isinstance(iface, dict):
+            self.netns = iface['netns']
+            logger.debug('udhcpc using netns %s', self.netns)
+
+        Popen = _nspopen_wrapper(self.netns) if self.netns else subprocess.Popen
+        bin_path = '/usr/bin/udhcpc6' if v6 else '/sbin/udhcpc'
+        cmdline = [bin_path, '-s', HANDLER_SCRIPT, '-i', iface['ifname'], '-f']
+        cmdline.append('-q' if once else '-R')
+        if hostname:
+            cmdline.append('-x')
+            if v6:
+                # TODO: We encode the fqdn for DHCPv6 because udhcpc6 seems to be broken
+                # flags: S bit set (see RFC4704)
+                enc_hostname = hostname.encode('utf-8')
+                enc_hostname = struct.pack('BB', 0b0001, len(enc_hostname)) + enc_hostname
+                enc_hostname = binascii.hexlify(enc_hostname).decode('ascii')
+                hostname_opt = f'0x27:{enc_hostname}'
+            else:
+                hostname_opt = f'hostname:{hostname}'
+            cmdline.append(hostname_opt)
+        if not v6:
+            cmdline += ['-V', VENDOR_ID]
+
+        self._suffix = '6' if v6 else ''
+        self._event_queue = posix_ipc.MessageQueue(f'/udhcpc{self._suffix}_{iface["address"].replace(":", "_")}', \
+            flags=os.O_CREAT | os.O_EXCL, max_messages=2, max_message_size=1024)
+        self.proc = Popen(cmdline, env={'EVENT_QUEUE': self._event_queue.name}, stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        if hostname:
+            logger.debug('[udhcpc%s#%d] using hostname "%s"', self._suffix, self.proc.pid, hostname)
+
+        self._has_lease = threading.Event()
+        self.ip = None
+        self.gateway = None
+        self.domain = None
+
+        self._shutdown_event = EventFD()
+        self.shutdown = False
+        self._event_thread = threading.Thread(target=self._read_events)
+        self._event_thread.start()
+
+    def _attr_listener(self, event_type, event):
+        if event_type in (EventType.BOUND, EventType.RENEW):
+            self.ip = ipaddress.ip_interface(event['ip'])
+            if 'gateway' in event:
+                self.gateway = ipaddress.ip_address(event['gateway'])
+            else:
+                self.gateway = None
+            self.domain = event.get('domain')
+            self._has_lease.set()
+        elif event_type == EventType.DECONFIG:
+            self._has_lease.clear()
+            self.ip = None
+            self.gateway = None
+            self.domain = None
+
+    def _read_events(self):
+        while True:
+            r, _w, _e = select([self._shutdown_event, self._event_queue.mqd], [], [])
+            if self._shutdown_event in r:
+                break
+
+            msg, _priority = self._event_queue.receive()
+            event = json.loads(msg.decode('utf-8'))
+            try:
+                event['type'] = EventType(event['type'])
+            except ValueError:
+                logger.warning('udhcpc%s#%d unknown event "%s"', self._suffix, self.proc.pid, event)
+                continue
+
+            logger.debug('[udhcp%s#%d event] %s', self._suffix, self.proc.pid, event)
+            for listener in self.event_listeners:
+                try:
+                    listener(self, event['type'], event)
+                except Exception as ex:
+                    logger.exception(ex)
+        self.shutdown = True
+        del self._shutdown_event
+
+    def await_ip(self, timeout=10):
+        if not self._has_lease.wait(timeout=timeout):
+            raise DHCPClientError(f'Timed out waiting for lease from udhcpc{self._suffix}')
+
+        return self.ip
+
+    def finish(self, timeout=5):
+        if self.shutdown or self._shutdown_event.is_set():
+            return
+
+        try:
+            if self.proc.returncode is not None and (not self.once or self.proc.returncode != 0):
+                raise DHCPClientError(f'udhcpc{self._suffix} exited early with code {self.proc.returncode}')
+            if self.once:
+                self.await_ip()
+            else:
+                self.proc.terminate()
+
+            if self.proc.wait(timeout=timeout) != 0:
+                raise DHCPClientError(f'udhcpc{self._suffix} exited with non-zero exit code {self.proc.returncode}')
+
+            return self.ip
+        finally:
+            self._shutdown_event.set()
+            self._event_thread.join()
+            self._event_queue.close()
+            self._event_queue.unlink()

net-dhcp/udhcpc_handler.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+import json
+import sys
+from os import environ as env
+
+import posix_ipc
+
+if __name__ != '__main__':
+    print('You shouldn\'t be importing this script!')
+    sys.exit(1)
+
+event = {'type': sys.argv[1]}
+if event['type'] in ('bound', 'renew'):
+    if 'ipv6' in env:
+        event['ip'] = env['ipv6']
+    else:
+        event['ip'] = f'{env["ip"]}/{env["mask"]}'
+        if 'router' in env:
+            event['gateway'] = env['router']
+        if 'domain' in env:
+            event['domain'] = env['domain']
+elif event['type'] in ('deconfig', 'leasefail', 'nak'):
+    pass
+else:
+    event['type'] = 'unknown'
+
+queue = posix_ipc.MessageQueue(env['EVENT_QUEUE'])
+queue.send(json.dumps(event))
+queue.close()

requirements.txt

@@ -0,0 +1,5 @@
+flask==1.1.1
+pyroute2==0.5.6
+docker==4.0.2
+eventfd==0.2
+posix_ipc==1.0.4