From 42c1c5fa66b042e1f2eac6812085a6ea44dad037 Mon Sep 17 00:00:00 2001 From: JDierkse Date: Sun, 24 Jan 2021 19:51:48 +0100 Subject: [PATCH] Initial commit --- .dockerignore | 1 + .gitignore | 4 + Dockerfile | 12 ++ Makefile | 40 ++++ README.md | 103 ++++++++++ config.json | 33 ++++ net-dhcp/__init__.py | 23 +++ net-dhcp/__main__.py | 14 ++ net-dhcp/interface.py | 203 +++++++++++++++++++ net-dhcp/network.py | 388 +++++++++++++++++++++++++++++++++++++ net-dhcp/udhcpc.py | 148 ++++++++++++++ net-dhcp/udhcpc_handler.py | 29 +++ requirements.txt | 5 + 13 files changed, 1003 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.md create mode 100644 config.json create mode 100644 net-dhcp/__init__.py create mode 100644 net-dhcp/__main__.py create mode 100644 net-dhcp/interface.py create mode 100644 net-dhcp/network.py create mode 100644 net-dhcp/udhcpc.py create mode 100755 net-dhcp/udhcpc_handler.py create mode 100644 requirements.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..87553b2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +/plugin/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f8f01be --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/plugin/ +__pycache__/ + +.vscode/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..85b41f8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.7-alpine + +COPY requirements.txt /opt/ +RUN apk --no-cache add gcc musl-dev && \ + pip install -r /opt/requirements.txt && \ + apk --no-cache del gcc musl-dev + +RUN mkdir -p /opt/plugin /run/docker/plugins /var/run/docker/netns +COPY net-dhcp/ /opt/plugin/net_dhcp + +WORKDIR /opt/plugin +ENTRYPOINT ["python", "-m", "net_dhcp"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..aebae0a --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +PLUGIN_NAME = registry.dierkse.nl/net-dhcp +PLUGIN_TAG ?= latest + +all: clean build rootfs create enable + +clean: + @echo "### rm ./plugin" + @rm -rf ./plugin + +build: + @echo "### docker build: rootfs image with net-dhcp" + @docker build -t ${PLUGIN_NAME}:rootfs . + +rootfs: + @echo "### create rootfs directory in ./plugin/rootfs" + @mkdir -p ./plugin/rootfs + @docker create --name tmp ${PLUGIN_NAME}:rootfs + @docker export tmp | tar -x -C ./plugin/rootfs + @echo "### copy config.json to ./plugin/" + @cp config.json ./plugin/ + @docker rm -vf tmp + +create: + @echo "### remove existing plugin ${PLUGIN_NAME}:${PLUGIN_TAG} if exists" + @docker plugin rm -f ${PLUGIN_NAME}:${PLUGIN_TAG} || true + @echo "### create new plugin ${PLUGIN_NAME}:${PLUGIN_TAG} from ./plugin" + @docker plugin create ${PLUGIN_NAME}:${PLUGIN_TAG} ./plugin + +debug: + @docker run --rm -ti --cap-add CAP_SYS_ADMIN --network host --volume /run/docker/plugins:/run/docker/plugins \ + --volume /run/docker.sock:/run/docker.sock --volume /var/run/docker/netns:/var/run/docker/netns \ + ${PLUGIN_NAME}:rootfs + +enable: + @echo "### enable plugin ${PLUGIN_NAME}:${PLUGIN_TAG}" + @docker plugin enable ${PLUGIN_NAME}:${PLUGIN_TAG} + +push: + @echo "### push plugin ${PLUGIN_NAME}:${PLUGIN_TAG}" + @docker plugin push ${PLUGIN_NAME}:${PLUGIN_TAG} diff --git a/README.md b/README.md new file mode 100644 index 0000000..0453bfc --- /dev/null +++ b/README.md @@ -0,0 +1,103 @@ +# docker-net-dhcp +`docker-net-dhcp` is a Docker plugin providing a network driver which allocates IP addresses (IPv4 and optionally IPv6) +via an existing DHCP server (e.g. your router). + +When configured correctly, this allows you to spin up a container (e.g. `docker run ...` or `docker-compose up ...`) and +access it on your network as if it was any other machine! + +# Usage +## Installation +``` +$ docker plugin install devplayer0/net-dhcp +Plugin "devplayer0/net-dhcp" is requesting the following privileges: + - network: [host] + - host pid namespace: [true] + - mount: [/var/run/docker.sock] + - capabilities: [CAP_NET_ADMIN CAP_SYS_ADMIN] +Do you grant the above permissions? [y/N] y +latest: Pulling from devplayer0/net-dhcp +: Download complete +Digest: sha256: +Status: Downloaded newer image for devplayer0/net-dhcp:latest +Installed plugin devplayer0/net-dhcp +$ +``` + +## Network creation +In order to create a Docker network using `net-dhcp`, you'll need a pre-configured bridge interface on the host. How you +set this up will depend on your system, but the following (manual) instructions should work on most Linux distros: +``` +# Create the bridge +$ sudo ip link add my-bridge type bridge +$ sudo ip link set my-bridge up + +# Assuming 'eth0' is connected to your LAN (where the DHCP server is) +$ sudo ip link set eth0 up +# Attach your network card to the bridge +$ sudo ip link set eth0 master my-bridge + +# Get an IP for the host (will go out to the DHCP server since eth0 is attached to the bridge) +$ sudo dhcpcd my-bridge +``` + +Once the bridge is ready, you can create the network: +``` +$ docker network create -d devplayer0/net-dhcp:latest --ipam-driver null -o bridge=my-bridge my-dhcp-net + +$ + +# With IPv6 enabled +# Although `docker network create` has a `--ipv6` flag, it doesn't work with the null IPAM driver +$ docker network create -d devplayer0/net-dhcp:latest --ipam-driver null -o bridge=test -o ipv6=true my-dhcp-net + +$ +``` +_Note: The `null` IPAM driver **must** be used, or else Docker will try to allocate IP addresses from its choice of +subnet - this can cause IP conflicts since the bridge is connected to your local network!_ + +## Container creation +Once you've created a network, you can create some containers: +``` +$ docker run --rm -ti --network my-dhcp-net alpine +/ # ip address show +1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1000 + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo + valid_lft forever preferred_lft forever +159: my-bridge0@if160: mtu 1500 qdisc noqueue state UP qlen 1000 + link/ether 86:41:68:f8:85:b9 brd ff:ff:ff:ff:ff:ff + inet 10.255.0.246/24 brd 10.255.0.255 scope global test0 + valid_lft forever preferred_lft forever +/ # ip route show +default via 10.255.0.123 dev my-bridge0 +10.255.0.0/24 dev my-bridge0 scope link src 10.255.0.246 +/ # +``` +Note: + - It will take a bit longer than usual for the container to start, as a DHCP lease needs to be obtained before creating it + - Once created, a persistent DHCP client will renew the DHCP lease (and then update the default gateway in the + container) when necessary - **this client runs separately from the container** + - Use `--mac-address` to specify a MAC address if you've configured reserved IP addresses on your DHCP server, or if + you want a container to re-use an old lease + +# Implementation +Fundamentally, the same mechanism is used by `net-dhcp` as Docker's `bridge` driver to wire up networking to containers. +That is, a bridge on the host is used as a switch so that containers can communicate with each other - `veth` pairs +connect each container's network namespace to the bridge. + +- While Docker creates and manages its own bridges (and routes and filters traffic), `net-dhcp` uses an existing bridge +on the host, bridged with the desired local network. +- Instead of allocating IP addresses from a static pool stored on the Docker host, `net-dhcp` relies on an external DHCP +server to provide IP addresses + +## Flow +1. Container creation request is made +2. A `veth` pair is created and the host end is connected to the bridge (at this point both interfaces are still in the +host namespace) +3. A DHCP client (BusyBox `udhcpc`) is started on the container end (still in the host namespace) - initial IP address +is provided to Docker by the plugin +4. Docker moves the container end of the `veth` pair into the container's network namespace and sets the IP address - at +this point `udhcpc` must be stopped +5. `net-dhcp` starts `udhcpc` on the container end of the `veth` pair in the container's **network namespace** (but +still in the host / plugin **PID namespace** - this means that the container can't see the DHCP client) +6. `udhcpc` continues to run, renewing the lease when required, until the container shuts down diff --git a/config.json b/config.json new file mode 100644 index 0000000..57f41db --- /dev/null +++ b/config.json @@ -0,0 +1,33 @@ +{ + "description": "Docker host bridge DHCP networking", + "interface": { + "socket": "net-dhcp.sock", + "types": [ + "docker.networkdriver/1.0" + ] + }, + "entrypoint": [ "python", "-m", "net_dhcp" ], + "workdir": "/opt/plugin", + "network": { + "type": "host" + }, + "mounts": [ + { + "source": "/var/run/docker.sock", + "destination": "/run/docker.sock", + "type": "bind", + "options": [ + "bind" + ] + } + ], + "pidhost": true, + "linux": { + "capabilities": [ + "CAP_NET_ADMIN", + "CAP_SYS_ADMIN", + "CAP_SYS_PTRACE", + "CAP_SYS_RESOURCE" + ] + } +} diff --git a/net-dhcp/__init__.py b/net-dhcp/__init__.py new file mode 100644 index 0000000..dbee061 --- /dev/null +++ b/net-dhcp/__init__.py @@ -0,0 +1,23 @@ +import logging + +from flask import Flask, jsonify + +class NetDhcpError(Exception): + def __init__(self, status, *args): + Exception.__init__(self, *args) + self.status = status + +app = Flask(__name__) + +from . import network + +logger = logging.getLogger('gunicorn.error') + +@app.errorhandler(404) +def err_not_found(_e): + return jsonify({'Err': 'API not found'}), 404 + +@app.errorhandler(Exception) +def err(e): + logger.exception(e) + return jsonify({'Err': str(e)}), 500 diff --git a/net-dhcp/__main__.py b/net-dhcp/__main__.py new file mode 100644 index 0000000..afda870 --- /dev/null +++ b/net-dhcp/__main__.py @@ -0,0 +1,14 @@ +import logging +import socketserver +from werkzeug.serving import run_simple +from . import app + +fh = logging.FileHandler('/var/log/net-dhcp.log') +fh.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')) + +logger = logging.getLogger('net-dhcp') +logger.setLevel(logging.DEBUG) +logger.addHandler(fh) + +socketserver.TCPServer.allow_reuse_address = True +run_simple('unix:///run/docker/plugins/net-dhcp.sock', 0, app) diff --git a/net-dhcp/interface.py b/net-dhcp/interface.py new file mode 100644 index 0000000..1968678 --- /dev/null +++ b/net-dhcp/interface.py @@ -0,0 +1,203 @@ +import atexit +import pyroute2 + +#------------------------------- + +def InitializeLibrary(library): + if library == 'NDB': + global ndb + ndb = pyroute2.NDB() + @atexit.register + def close_ndb(): + ndb.close() + elif library == 'IPR': + global iproute + iproute = pyroute2.IPRoute() + @atexit.register + def close_iproute(): + iproute.close() + +#------------------------------- + +def GetInterfaces(library): + interfaces = [] + if library == 'NDB': + for itf in ndb.interfaces: + interfaces.append(Interface(ndb.interfaces[itf.ifname])) + elif library == 'IPR': + for itf in iproute.get_links(): + interfaces.append(Interface(itf)) + return interfaces + +def GetInterface(library, ifname): + if library == 'NDB': + return Interface(ndb.interfaces[ifname]) + elif library == 'IPR': + return Interface(iproute.get_links(ifname=ifname)[0]) + return None + +class Interface: + def __init__(self, interface): + self.ndb = None + self.ipr = None + + self._set_interface_(interface) + self._initialize_attributes_() + + def __getitem__(self, key): + return getattr(self, key, None) + + def _set_interface_(self, interface): + if isinstance(interface, pyroute2.netlink.rtnl.ifinfmsg.ifinfmsg): + self.ipr = interface + elif isinstance(interface, pyroute2.ndb.objects.interface.Interface): + self.ndb = interface + + def _initialize_attributes_(self): + if self.ndb: + self.index = self.ndb['index'] + self.ifname = self.ndb['ifname'] + self.kind = self.ndb['kind'] + self.address = self.ndb['address'] + if 'netns' in self.ndb: + self.netns = self.ndb['netns'] + else: + self.netns = None + self.ipaddr = [] + for address in self.ndb.ipaddr: + ipaddress = {} + ipaddress['address'] = address['address'] + ipaddress['prefixlen'] = address['prefixlen'] + self.ipaddr.append(ipaddress) + self.routes = [] + for r in self.ndb.routes: + route = {} + route['type'] = r['type'] + route['family'] = r['family'] + route['dst'] = r['dst'] + route['gateway'] = r['gateway'] + route['dst_len'] = r['dst_len'] + self.routes.append(route) + elif self.ipr: + self.index = self.ipr['index'] + self.ifname = self.ipr.get_attr('IFLA_IFNAME') + self.kind = self.ipr.get_attr('IFLA_LINKINFO').get_attr('IFLA_INFO_KIND') if self.ipr.get_attr('IFLA_LINKINFO') else None + self.address = self.ipr.get_attr('IFLA_ADDRESS') + self.netns = None + self.ipaddr = [] + for address in iproute.get_addr(label=self.ipr.get_attr('IFLA_IFNAME')): + ipaddress = {} + ipaddress['address'] = address.get_attr('IFA_ADDRESS') + ipaddress['prefixlen'] = address['prefixlen'] + self.ipaddr.append(ipaddress) + self.routes = [] + for r in iproute.get_routes(): + attrs = dict(r['attrs']) + if attrs['RTA_OIF'] == self['index']: + if 'RTA_PRIORITY' in attrs and not attrs['RTA_PRIORITY'] < 255: + continue + route = {} + route['type'] = r['type'] + route['family'] = r['family'] + route['dst'] = attrs['RTA_DST'] if 'RTA_DST' in attrs else '' + route['gateway'] = attrs['RTA_GATEWAY'] if 'RTA_GATEWAY' in attrs else None + route['dst_len'] = r['dst_len'] + self.routes.append(route) + + def up(self): + if self.ndb: + self.ndb.set('state', 'up').commit() + elif self.ipr: + iproute.link("set", index=self.index, state="up") + + def down(self): + if self.ndb: + self.ndb.set('state', 'down').commit() + elif self.ipr: + iproute.link("set", index=self.index, state="down") + +def CreateInterface(library, ifname, kind, peer): + try: + if library == 'NDB': + ndb.interfaces.create(ifname=ifname, kind=kind, peer=peer).commit() + return Interface(ndb.interfaces[ifname]) + elif library == 'IPR': + iproute.link("add", ifname=ifname, kind=kind, peer=peer) + return Interface(iproute.get_links(ifname=ifname)[0]) + return None + except Exception as e: + return None + +def RemoveInterface(library, ifname): + try: + if library == 'NDB': + ndb.interfaces[ifname].remove().commit() + elif library == 'IPR': + iproute.link("del", index=iproute.link_lookup(ifname=ifname)[0]) + except Exception as e: + return + +def AddPort(library, bridge, ifname): + try: + if library == 'NDB': + ndb.interfaces[bridge].add_port(ifname).commit() + elif library == 'IPR': + iproute.link("set", index=iproute.link_lookup(ifname=ifname)[0], master=iproute.link_lookup(ifname=bridge)[0]) + except Exception as e: + return + +def DelPort(library, bridge, ifname): + try: + if library == 'NDB': + ndb.interfaces[bridge].del_port(ifname).commit() + elif library == 'IPR': + iproute.link("set", index=iproute.link_lookup(ifname=ifname)[0], master=0) + except Exception as e: + return + +#------------------------------- + +def test_Variable(v1, v2, f): + assert v1 == v2, "{} not equal ({}, {})".format(f, v1, v2) + +def test_Interface(): + itf1 = GetInterface('NDB', 'br0') + itf2 = GetInterface('IPR', 'br0') + + test_Variable(itf1.ifname, itf2.ifname, 'ifname') + test_Variable(itf1.kind, itf2.kind, 'kind') + test_Variable(itf1.ipaddr(), itf2.ipaddr(), 'ipaddr') + test_Variable(itf1.routes, itf2.routes, 'routes') + + print("OK") +#test_Interface() + +def test_CreateInterface(): + CreateInterface('NDB', 'ndb0', 'veth', 'ndb1') + CreateInterface('IPR', 'ipr0', 'veth', 'ipr1') + + itf1 = GetInterface('NDB', 'ndb0') + itf2 = GetInterface('IPR', 'ipr0') + + itf1.up(); + itf2.up(); + + itf3 = GetInterface('NDB', 'ndb1') + itf4 = GetInterface('IPR', 'ipr1') + + itf3.up(); + itf4.up(); + + AddPort('NDB', 'br0', 'ndb0') + AddPort('IPR', 'br0', 'ipr0') + +#test_CreateInterface() + +def test_RemoveInterface(): + DelPort('NDB', 'br0', 'ndb0') + DelPort('IPR', 'br0', 'ipr0') + + RemoveInterface('NDB', 'ndb0') + RemoveInterface('IPR', 'ipr0') +#test_RemoveInterface() + diff --git a/net-dhcp/network.py b/net-dhcp/network.py new file mode 100644 index 0000000..8a715cf --- /dev/null +++ b/net-dhcp/network.py @@ -0,0 +1,388 @@ +import itertools +import ipaddress +import logging +import atexit +import socket +import time +import threading +import subprocess + +import pyroute2 +from pyroute2.netlink.rtnl import rtypes +import docker +from flask import request, jsonify + +from . import interface +from . import NetDhcpError, udhcpc, app + +LIBRARY = 'IPR' + +OPTS_KEY = 'com.docker.network.generic' +OPT_BRIDGE = 'bridge' +OPT_IPV6 = 'ipv6' + +logger = logging.getLogger('net-dhcp') + +interface.InitializeLibrary(LIBRARY) + +client = docker.from_env() +@atexit.register +def close_docker(): + client.close() + +gateway_hints = {} +container_dhcp_clients = {} +@atexit.register +def cleanup_dhcp(): + for endpoint, dhcp in container_dhcp_clients.items(): + logger.warning('cleaning up orphaned container DHCP client (endpoint "%s")', endpoint) + dhcp.stop() + +def veth_pair(e): + return f'dh-{e[:12]}', f'{e[:12]}-dh' + +def iface_addrs(iface): + return list(map(lambda a: ipaddress.ip_interface((a['address'], a['prefixlen'])), iface.ipaddr)) +def iface_nets(iface): + return list(map(lambda n: n.network, iface_addrs(iface))) + +def get_bridges(): + reserved_nets = set(map(ipaddress.ip_network, map(lambda c: c['Subnet'], \ + itertools.chain.from_iterable(map(lambda i: i['Config'], filter(lambda i: i['Driver'] != 'net-dhcp', \ + map(lambda n: n.attrs['IPAM'], client.networks.list()))))))) + + return dict(map(lambda i: (i['ifname'], i), filter(lambda i: i['kind'] == 'bridge' and not \ + set(iface_nets(i)).intersection(reserved_nets), map(lambda i: interface.GetInterface(LIBRARY, i['ifname']), \ + interface.GetInterfaces(LIBRARY))))) + +def net_bridge(n): + return interface.GetInterface(LIBRARY, client.networks.get(n).attrs['Options'][OPT_BRIDGE]) +def ipv6_enabled(n): + options = client.networks.get(n).attrs['Options'] + return OPT_IPV6 in options and options[OPT_IPV6] == 'true' + +def endpoint_container_iface(n, e): + for cid, info in client.networks.get(n).attrs['Containers'].items(): + if info['EndpointID'] == e: + container = client.containers.get(cid) + netns = f'/proc/{container.attrs["State"]["Pid"]}/ns/net' + + with pyroute2.NetNS(netns) as rtnl: + for link in rtnl.get_links(): + attrs = dict(link['attrs']) + if attrs['IFLA_ADDRESS'] == info['MacAddress']: + return { + 'netns': netns, + 'ifname': attrs['IFLA_IFNAME'], + 'address': attrs['IFLA_ADDRESS'] + } + break + return None +def await_endpoint_container_iface(n, e, timeout=5): + start = time.time() + iface = None + while time.time() - start < timeout: + try: + iface = endpoint_container_iface(n, e) + except docker.errors.NotFound: + time.sleep(0.5) + if not iface: + raise NetDhcpError('Timed out waiting for container to become availabile') + return iface + +def endpoint_container_hostname(n, e): + for cid, info in client.networks.get(n).attrs['Containers'].items(): + if info['EndpointID'] == e: + return client.containers.get(cid).attrs['Config']['Hostname'] + return None + +@app.route('/NetworkDriver.GetCapabilities', methods=['POST']) +def net_get_capabilities(): + return jsonify({ + 'Scope': 'local', + 'ConnectivityScope': 'global' + }) + +@app.route('/NetworkDriver.CreateNetwork', methods=['POST']) +def create_net(): + req = request.get_json(force=True) + for data in req['IPv4Data']: + if data['AddressSpace'] != 'null' or data['Pool'] != '0.0.0.0/0': + return jsonify({'Err': 'Only the null IPAM driver is supported'}), 400 + + options = req['Options'][OPTS_KEY] + if OPT_BRIDGE not in options: + return jsonify({'Err': 'No bridge provided'}), 400 + # We have to use a custom "enable IPv6" option because Docker's null IPAM driver doesn't support IPv6 and a plugin + # IPAM driver isn't allowed to return an empty address + if OPT_IPV6 in options and options[OPT_IPV6] not in ('', 'true', 'false'): + return jsonify({'Err': 'Invalid boolean value for ipv6'}), 400 + + desired = options[OPT_BRIDGE] + bridges = get_bridges() + if desired not in bridges: + return jsonify({'Err': f'Bridge "{desired}" not found (or the specified bridge is already used by Docker)'}), 400 + + logger.info('Creating network "%s" (using bridge "%s")', req['NetworkID'], desired) + return jsonify({}) + +@app.route('/NetworkDriver.DeleteNetwork', methods=['POST']) +def delete_net(): + return jsonify({}) + +@app.route('/NetworkDriver.CreateEndpoint', methods=['POST']) +def create_endpoint(): + req = request.get_json(force=True) + network_id = req['NetworkID'] + endpoint_id = req['EndpointID'] + req_iface = req['Interface'] + + bridge = net_bridge(network_id) + bridge_addrs = iface_addrs(bridge) + + if_host, if_container = veth_pair(endpoint_id) + logger.info('creating veth pair %s <=> %s', if_host, if_container) + if_host = interface.CreateInterface(LIBRARY, if_host, 'veth', if_container) + if_host.up() + + try: + start = time.time() + while isinstance(if_container, str) and time.time() - start < 10: + try: + if_container = interface.GetInterface(LIBRARY, if_container) + if_container.up() + except KeyError: + time.sleep(0.5) + if isinstance(if_container, str): + raise NetDhcpError(f'timed out waiting for {if_container} to appear in host') + + interface.AddPort(LIBRARY, bridge.ifname, if_host.ifname) + + res_iface = { + 'MacAddress': '', + 'Address': '', + 'AddressIPv6': '' + } + +# !!! +# if 'MacAddress' in req_iface and req_iface['MacAddress']: +# (if_container +# .set('address', req_iface['MacAddress']) +# .commit()) +# else: +# res_iface['MacAddress'] = if_container['address'] +# !!! + res_iface['MacAddress'] = if_container['address'] + + def try_addr(type_): + addr = None + k = 'AddressIPv6' if type_ == 'v6' else 'Address' + if k in req_iface and req_iface[k]: + # TODO: Should we allow static IP's somehow? + # Just validate the address, Docker will add it to the interface for us + #addr = ipaddress.ip_interface(req_iface[k]) + #for bridge_addr in bridge_addrs: + # if addr.ip == bridge_addr.ip: + # raise NetDhcpError(400, f'Address {addr} is already in use on bridge {bridge["ifname"]}') + raise NetDhcpError('Only the null IPAM driver is supported') + else: + dhcp = udhcpc.DHCPClient(if_container, v6=type_ == 'v6', once=True) + addr = dhcp.finish() + if not addr: + return + res_iface[k] = str(addr) + + if dhcp.gateway: + gateway_hints[endpoint_id] = dhcp.gateway + logger.info('Adding IP%s address %s to %s', type_, addr, if_container['ifname']) + + try_addr('v4') + if ipv6_enabled(network_id): + try_addr('v6') + + res = jsonify({ + 'Interface': res_iface + }) + except Exception as e: + logger.exception(e) + + if not isinstance(if_container, str): + interface.DelPort(LIBRARY, bridge.ifname, if_host.ifname) + interface.RemoveInterface(LIBRARY, if_host.ifname) + + if isinstance(e, NetDhcpError): + res = jsonify({'Err': str(e)}), e.status + else: + res = jsonify({'Err': str(e)}), 500 + finally: + return res + +@app.route('/NetworkDriver.EndpointOperInfo', methods=['POST']) +def endpoint_info(): + req = request.get_json(force=True) + + bridge = net_bridge(req['NetworkID']) + if_host, _if_container = veth_pair(req['EndpointID']) + if_host = interface.GetInterface(LIBRARY, if_host) + + return jsonify({ + 'bridge': bridge['ifname'], + 'if_host': { + 'name': if_host['ifname'], + 'mac': if_host['address'] + } + }) + +@app.route('/NetworkDriver.DeleteEndpoint', methods=['POST']) +def delete_endpoint(): + req = request.get_json(force=True) + + bridge = net_bridge(req['NetworkID']) + if_host, _if_container = veth_pair(req['EndpointID']) + if_host = interface.GetInterface(LIBRARY, if_host) + + interface.DelPort(LIBRARY, bridge.ifname, if_host.ifname) + interface.RemoveInterface(LIBRARY, if_host.ifname) + + return jsonify({}) + +@app.route('/NetworkDriver.Join', methods=['POST']) +def join(): + req = request.get_json(force=True) + network = req['NetworkID'] + endpoint = req['EndpointID'] + + bridge = net_bridge(req['NetworkID']) + _if_host, if_container = veth_pair(req['EndpointID']) + + res = { + 'InterfaceName': { + 'SrcName': if_container, + 'DstPrefix': bridge['ifname'] + }, + 'StaticRoutes': [] + } + + if endpoint in gateway_hints: + gateway = gateway_hints[endpoint] + logger.info('Setting IPv4 gateway from DHCP (%s)', gateway) + res['Gateway'] = str(gateway) + del gateway_hints[endpoint] + + ipv6 = ipv6_enabled(network) + for route in bridge.routes: + if route['type'] != rtypes['RTN_UNICAST'] or \ + (route['family'] == socket.AF_INET6 and not ipv6): + continue + + if route['dst'] in ('', '/0'): + if route['family'] == socket.AF_INET and 'Gateway' not in res: + logger.info('Adding IPv4 gateway %s', route['gateway']) + res['Gateway'] = route['gateway'] + elif route['family'] == socket.AF_INET6 and 'GatewayIPv6' not in res: + logger.info('Adding IPv6 gateway %s', route['gateway']) + res['GatewayIPv6'] = route['gateway'] + elif route['gateway']: + dst = f'{route["dst"]}/{route["dst_len"]}' + logger.info('Adding route to %s via %s', dst, route['gateway']) + res['StaticRoutes'].append({ + 'Destination': dst, + 'RouteType': 0, + 'NextHop': route['gateway'] + }) + + container_dhcp_clients[endpoint] = ContainerDHCPManager(network, endpoint) + return jsonify(res) + +@app.route('/NetworkDriver.Leave', methods=['POST']) +def leave(): + req = request.get_json(force=True) + endpoint = req['EndpointID'] + + if endpoint in container_dhcp_clients: + container_dhcp_clients[endpoint].stop() + del container_dhcp_clients[endpoint] + + return jsonify({}) + +# Trying to grab the container's attributes (to get the network namespace) +# will deadlock (since Docker is waiting on us), so we must defer starting +# the DHCP client +class ContainerDHCPManager: + def __init__(self, network, endpoint): + self.network = network + self.endpoint = endpoint + self.ipv6 = ipv6_enabled(network) + + self.dhcp = None + self.dhcp6 = None + self._thread = threading.Thread(target=self.run) + self._thread.start() + + def _on_event(self, dhcp, event_type, _event): + if event_type != udhcpc.EventType.RENEW or not dhcp.gateway: + return + + logger.info('[dhcp container] Replacing gateway with %s', dhcp.gateway) + subprocess.check_call(['nsenter', f'-n{dhcp.netns}', '--', '/sbin/ip', 'route', 'replace', 'default', 'via', + str(dhcp.gateway)], timeout=1, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + # TODO: Adding default route with NDB seems to be broken (because of the dst syntax?) + #for route in ndb.routes: + # if route['type'] != rtypes['RTN_UNICAST'] or \ + # route['oif'] != dhcp.iface['index'] or \ + # (route['family'] == socket.AF_INET6 and not self.ipv6) or \ + # route['dst'] not in ('', '/0'): + # continue + + # # Needed because Route.remove() doesn't like a blank destination + # logger.info('Removing default route via %s', route['gateway']) + # route['dst'] = '::' if route['family'] == socket.AF_INET6 else '0.0.0.0' + # (route + # .remove() + # .commit()) + + #logger.info('Adding default route via %s', dhcp.gateway) + #(ndb.routes.add({'oif': dhcp.iface['index'], 'gateway': dhcp.gateway}) + # .commit()) + + def run(self): + try: + iface = await_endpoint_container_iface(self.network, self.endpoint) + hostname = endpoint_container_hostname(self.network, self.endpoint) + + self.dhcp = udhcpc.DHCPClient(iface, event_listener=self._on_event, hostname=hostname) + logger.info('Starting DHCPv4 client on %s in container namespace %s', iface['ifname'], \ + self.dhcp.netns) + + if self.ipv6: + self.dhcp6 = udhcpc.DHCPClient(iface, v6=True, event_listener=self._on_event, hostname=hostname) + logger.info('Starting DHCPv6 client on %s in container namespace %s', iface['ifname'], \ + self.dhcp6.netns) + except Exception as e: + logger.exception(e) + if self.dhcp: + self.dhcp.finish(timeout=1) + + def stop(self): + if not self.dhcp: + return + + try: + logger.info('Shutting down DHCPv4 client on %s in container namespace %s', \ + self.dhcp.iface['ifname'], self.dhcp.netns) + self.dhcp.finish(timeout=1) + finally: + try: + if self.ipv6: + logger.info('Shutting down DHCPv6 client on %s in container namespace %s', \ + self.dhcp6.iface['ifname'], self.dhcp.netns) + self.dhcp6.finish(timeout=1) + finally: + self._thread.join() + + # we have to do this since the docker client leaks sockets... + global client + client.close() + client = docker.from_env() diff --git a/net-dhcp/udhcpc.py b/net-dhcp/udhcpc.py new file mode 100644 index 0000000..55d1f2b --- /dev/null +++ b/net-dhcp/udhcpc.py @@ -0,0 +1,148 @@ +from enum import Enum +import ipaddress +import json +import struct +import binascii +import os +from os import path +from select import select +import threading +import subprocess +import logging + +from eventfd import EventFD +import posix_ipc + +HANDLER_SCRIPT = path.join(path.dirname(__file__), 'udhcpc_handler.py') +AWAIT_INTERVAL = 0.1 +VENDOR_ID = 'docker' + +class EventType(Enum): + BOUND = 'bound' + RENEW = 'renew' + DECONFIG = 'deconfig' + LEASEFAIL = 'leasefail' + +logger = logging.getLogger('net-dhcp') + +class DHCPClientError(Exception): + pass + +def _nspopen_wrapper(netns): + return lambda cmd, *args, **kwargs: subprocess.Popen(['nsenter', f'-n{netns}', '--'] + cmd, *args, **kwargs) +class DHCPClient: + def __init__(self, iface, v6=False, once=False, hostname=None, event_listener=None): + self.iface = iface + self.v6 = v6 + self.once = once + self.event_listeners = [DHCPClient._attr_listener] + if event_listener: + self.event_listeners.append(event_listener) + + self.netns = None + if isinstance(iface, dict): + self.netns = iface['netns'] + logger.debug('udhcpc using netns %s', self.netns) + + Popen = _nspopen_wrapper(self.netns) if self.netns else subprocess.Popen + bin_path = '/usr/bin/udhcpc6' if v6 else '/sbin/udhcpc' + cmdline = [bin_path, '-s', HANDLER_SCRIPT, '-i', iface['ifname'], '-f'] + cmdline.append('-q' if once else '-R') + if hostname: + cmdline.append('-x') + if v6: + # TODO: We encode the fqdn for DHCPv6 because udhcpc6 seems to be broken + # flags: S bit set (see RFC4704) + enc_hostname = hostname.encode('utf-8') + enc_hostname = struct.pack('BB', 0b0001, len(enc_hostname)) + enc_hostname + enc_hostname = binascii.hexlify(enc_hostname).decode('ascii') + hostname_opt = f'0x27:{enc_hostname}' + else: + hostname_opt = f'hostname:{hostname}' + cmdline.append(hostname_opt) + if not v6: + cmdline += ['-V', VENDOR_ID] + + self._suffix = '6' if v6 else '' + self._event_queue = posix_ipc.MessageQueue(f'/udhcpc{self._suffix}_{iface["address"].replace(":", "_")}', \ + flags=os.O_CREAT | os.O_EXCL, max_messages=2, max_message_size=1024) + self.proc = Popen(cmdline, env={'EVENT_QUEUE': self._event_queue.name}, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if hostname: + logger.debug('[udhcpc%s#%d] using hostname "%s"', self._suffix, self.proc.pid, hostname) + + self._has_lease = threading.Event() + self.ip = None + self.gateway = None + self.domain = None + + self._shutdown_event = EventFD() + self.shutdown = False + self._event_thread = threading.Thread(target=self._read_events) + self._event_thread.start() + + def _attr_listener(self, event_type, event): + if event_type in (EventType.BOUND, EventType.RENEW): + self.ip = ipaddress.ip_interface(event['ip']) + if 'gateway' in event: + self.gateway = ipaddress.ip_address(event['gateway']) + else: + self.gateway = None + self.domain = event.get('domain') + self._has_lease.set() + elif event_type == EventType.DECONFIG: + self._has_lease.clear() + self.ip = None + self.gateway = None + self.domain = None + + def _read_events(self): + while True: + r, _w, _e = select([self._shutdown_event, self._event_queue.mqd], [], []) + if self._shutdown_event in r: + break + + msg, _priority = self._event_queue.receive() + event = json.loads(msg.decode('utf-8')) + try: + event['type'] = EventType(event['type']) + except ValueError: + logger.warning('udhcpc%s#%d unknown event "%s"', self._suffix, self.proc.pid, event) + continue + + logger.debug('[udhcp%s#%d event] %s', self._suffix, self.proc.pid, event) + for listener in self.event_listeners: + try: + listener(self, event['type'], event) + except Exception as ex: + logger.exception(ex) + self.shutdown = True + del self._shutdown_event + + def await_ip(self, timeout=10): + if not self._has_lease.wait(timeout=timeout): + raise DHCPClientError(f'Timed out waiting for lease from udhcpc{self._suffix}') + + return self.ip + + def finish(self, timeout=5): + if self.shutdown or self._shutdown_event.is_set(): + return + + try: + if self.proc.returncode is not None and (not self.once or self.proc.returncode != 0): + raise DHCPClientError(f'udhcpc{self._suffix} exited early with code {self.proc.returncode}') + if self.once: + self.await_ip() + else: + self.proc.terminate() + + if self.proc.wait(timeout=timeout) != 0: + raise DHCPClientError(f'udhcpc{self._suffix} exited with non-zero exit code {self.proc.returncode}') + + return self.ip + finally: + self._shutdown_event.set() + self._event_thread.join() + self._event_queue.close() + self._event_queue.unlink() diff --git a/net-dhcp/udhcpc_handler.py b/net-dhcp/udhcpc_handler.py new file mode 100755 index 0000000..65374aa --- /dev/null +++ b/net-dhcp/udhcpc_handler.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +import json +import sys +from os import environ as env + +import posix_ipc + +if __name__ != '__main__': + print('You shouldn\'t be importing this script!') + sys.exit(1) + +event = {'type': sys.argv[1]} +if event['type'] in ('bound', 'renew'): + if 'ipv6' in env: + event['ip'] = env['ipv6'] + else: + event['ip'] = f'{env["ip"]}/{env["mask"]}' + if 'router' in env: + event['gateway'] = env['router'] + if 'domain' in env: + event['domain'] = env['domain'] +elif event['type'] in ('deconfig', 'leasefail', 'nak'): + pass +else: + event['type'] = 'unknown' + +queue = posix_ipc.MessageQueue(env['EVENT_QUEUE']) +queue.send(json.dumps(event)) +queue.close() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..50bf149 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +flask==1.1.1 +pyroute2==0.5.6 +docker==4.0.2 +eventfd==0.2 +posix_ipc==1.0.4