initial public commit

_modules/ffsaar.py

+from salt.utils.network import ipaddress
+import itertools, copy
+
+'''
+Python module for our salt state.  Mostly performs IP address computations.
+There are several "magic" pillar variable that this accesses:
+- `servername` is used as a default for various functions that compute
+  the address of a particular server.
+- `servers` is used as a list of all servers.
+- `domains` is used as a list of all domains.
+'''
+
+## Global (non-domain-specific) functions
+def is_gw(name = None):
+    if name is None: name = __pillar__["servername"]
+    return name.startswith("gw")
+
+def gw_num(name = None):
+    if name is None: name = __pillar__["servername"]
+    assert is_gw(name)
+    gw = int(name[len("gw"):])
+    assert gw > 0
+    return gw
+
+def tunip_local(tunnel):
+    '''Local GRE tunnel address for this tunnel (given as a dict)'''
+    return str(ipaddress.IPv4Address(tunnel['tunip_remote']) + 1)
+
+def tunip_local_6(tunnel):
+    '''Local GRE tunnel address for this tunnel (given as a dict)'''
+    return str(ipaddress.IPv6Address(tunnel['tunip_remote_6']) + 1)
+
+def tunneldigger_ports(test):
+    '''Returns a comma-separated list of all tunneldigger ports of all domains'''
+    return ",".join(itertools.chain(*map(
+        lambda domain: map(str, domain['tunneldigger_test_ports' if test else 'tunneldigger_ports']),
+        __pillar__["domains"]
+    )))
+
+def server_pubip(name = None):
+    '''Internet IPv4 address of this server'''
+    if name is None: name = __pillar__["servername"]
+    for gw in __pillar__["servers"]:
+        if gw['name'] == name:
+            return gw['ip']
+    raise Exception("Server with name `{}` not found".format(name))
+
+def server_pubip_6(name = None):
+    '''Internet IPv6 address of this server'''
+    if name is None: name = __pillar__["servername"]
+    for gw in __pillar__["servers"]:
+        if gw['name'] == name:
+            return gw['ip_6']
+    raise Exception("Server with name `{}` not found".format(name))
+
+def mac2ipv6_host(mac):
+    # Taken from https://stackoverflow.com/questions/37140846/how-to-convert-ipv6-link-local-address-to-mac-address-in-python
+    # only accept MACs separated by a colon
+    parts = str(mac).split(":")
+    assert len(parts) == 6
+
+    # modify parts to match IPv6 value
+    parts.insert(3, "ff")
+    parts.insert(4, "fe")
+    parts[0] = "%x" % (int(parts[0], 16) ^ 2)
+
+    # format output
+    ipv6Parts = []
+    for i in range(0, len(parts), 2):
+        ipv6Parts.append("".join(parts[i:i+2]))
+    return ipaddress.IPv6Address('::' + ":".join(ipv6Parts))
+
+def increment_mac(mac, inc):
+    '''Increment `mac` (a string) by `inc` (an int)'''
+    mac_parts = mac.split(':')
+    mac_parts[5] = ("%x" % ((int(mac_parts[5]) + inc) % 256)).zfill(2)
+    return ':'.join(mac_parts)
+
+## Domain-specific functions
+def domains():
+    return list(map(lambda domaininfo: Domain(domaininfo), __pillar__["domains"]))
+
+def domain(name):
+    for domain in __pillar__["domains"]:
+        if domain['name'] == name:
+            return Domain(domain)
+    raise Exception("Domain with name `{}` not found".format(name))
+
+def fill_with_defaults(local, defaults):
+    '''mutates `local`!'''
+    for key in defaults:
+        val = defaults[key]
+        if key not in local:
+            # easy, a copy
+            local[key] = val
+        elif isinstance(val, dict):
+            # default is a dict, merge with local
+            assert isinstance(local[key], dict)
+            fill_with_defaults(local[key], val)
+
+class Domain:
+    def __init__(self, domaininfo):
+        domaininfo = copy.deepcopy(domaininfo)
+        fill_with_defaults(domaininfo, __pillar__["domain_defaults"])
+        for key in domaininfo:
+            setattr(self, key, domaininfo[key])
+
+    def offset(self, name):
+        if name is None: name = __pillar__["servername"]
+        if is_gw(name):
+            return self.gw_base_offset + gw_num(name)
+        else:
+            return getattr(self, name + "_offset")
+
+    def intranet_mac(self):
+        return increment_mac(self.mac['intranet_base'], self.offset(None))
+
+    def batman_mac(self):
+        return increment_mac(self.mac['batman_base'], self.offset(None))
+
+    def net(self):
+        return ipaddress.IPv4Network(self.network['addr'])
+
+    def host(self, name = None):
+        '''Compute the IPv4 address of the current host in this domain'''
+        host = self.net().network_address + self.offset(name)
+        assert host in self.net()
+        return host
+
+    def net_6(self):
+        return ipaddress.IPv6Network(self.network_6['addr'])
+
+    def host_6(self, name = None):
+        '''Compute the IPv6 address of the current host in this domain'''
+        return self.net_6().network_address + self.offset(name)
+
+    def ffrl_net_6(self):
+        return ipaddress.IPv6Network(self.ffrl_6['addr'])
+
+    def ffrl_host_6(self, name = None):
+        '''Compute the public FFRL IPv6 address of the host in this domain'''
+        return self.ffrl_net_6().network_address + self.offset(name)
+
+    def ffrl_subnet_6(self, n):
+        '''Return the n-th subnet of our FFRL space'''
+        base = self.ffrl_net_6().network_address
+        prefixlen = self.ffrl_6['subnet_prefixlen']
+        base += n * (1 << (128 - prefixlen)) # the n-th /prefixlen network
+        return ipaddress.IPv6Network("{}/{}".format(base, prefixlen))
+
+    def ffrl_servers_net_6(self):
+        return self.ffrl_subnet_6(0)
+
+    def ffrl_clients_net_6(self, name = None):
+        return self.ffrl_subnet_6(gw_num(name))
+
+    def ffrl_userservices_net_6(self):
+        return self.ffrl_subnet_6(self.ffrl_6['userservices_subnet'])
+
+    def userservice(self, offset):
+        host = self.net().network_address + self.network['user_services_base_offset'] + offset
+        assert host in self.net()
+        return host
+
+    def userservice_6(self, mac):
+        return self.ffrl_userservices_net_6().network_address + int(mac2ipv6_host(mac))
+
+    def dhcp_base(self):
+        base = self.net().network_address
+        dhcp = self.network.get('dhcp', __pillar__["domain_defaults"]["network"]["dhcp"])
+        base += dhcp['base_offset']
+        base += (gw_num() - 1) * dhcp['stride']
+        assert base in self.net()
+        return base
+
+    def dhcp_ceil(self):
+        dhcp = self.network.get('dhcp', __pillar__["domain_defaults"]["network"]["dhcp"])
+        ceil = self.dhcp_base() + (dhcp['block_size'] - 1)
+        assert ceil in self.net()
+        return ceil

freifunk-common/batman.sls

+batman_dep:
+  pkg.latest:
+    - pkgs:
+      - build-essential
+      - pkg-config
+      - checkinstall
+      # batctl
+      - libnl-3-dev
+      - libnl-genl-3-dev
+      # batan_adv
+      - linux-headers-amd64
+      - dkms
+
+batman-adv_git:
+  git.latest:
+    - name: {{ pillar.batman_adv.url }}
+    - target: /usr/src/batman-adv-{{ pillar.batman_adv.version }}
+    - rev: {{ pillar.batman_adv.commit }}
+    # allow non-fast-forward
+    - force_fetch: True
+    - force_reset: True
+    - require:
+      - pkg: git
+
+/usr/src/batman-adv-{{ pillar.batman_adv.version }}/dkms.conf:
+  file.managed:
+    - source: salt://freifunk-common/files/batman/dkms.conf
+    - template: jinja
+    - require:
+      - git: batman-adv_git
+
+batman-adv_dkms:
+  cmd.script:
+    - name: salt://freifunk-common/files/batman/install-dkms.sh
+    - template: jinja
+    - shell: /bin/bash
+    - onchanges:
+      - git: batman-adv_git
+      - file: /usr/src/batman-adv-{{ pillar.batman_adv.version }}/dkms.conf
+    - require:
+      - pkg: batman_dep
+
+batman_adv:
+  kmod.present:
+    - persist: True
+
+/usr/local/sbin/build-batctl.sh:
+  file.absent
+
+build_batctl:
+  cmd.script:
+    - name: salt://freifunk-common/files/batman/build-batctl.sh
+    - template: jinja
+    - shell: /bin/bash
+    - unless: "[ \"$(batctl -v | awk '{ print $2 }')\" = \"{{ pillar.batctl.version }}\" ]"
+    - require:
+      - pkg: git
+      - pkg: batman_dep

freifunk-common/files/batman/build-batctl.sh

+#!/bin/bash
+set -e
+
+PACKAGE="batctl"
+REMOTE="{{ pillar.batctl.url }}"
+TAG="{{ pillar.batctl.commit }}"
+TARGET_VERSION="{{ pillar.batctl.version }}"
+BUILD_ROOT="/usr/src/batctl"
+
+prepare() {
+	rm -rf $BUILD_ROOT
+	git clone "$REMOTE" "$BUILD_ROOT"
+	cd $BUILD_ROOT
+	git checkout "$TAG"
+}
+
+build() {
+	cd $BUILD_ROOT
+	make -j$(nproc)
+}
+
+install() {
+	cd $BUILD_ROOT
+	checkinstall -y --nodoc --pkgname "${PACKAGE}" --pkgversion "${TARGET_VERSION}" --pkgrelease 9 --fstrans=no
+}
+
+prepare
+build
+install

freifunk-common/files/batman/dkms.conf

+PACKAGE_NAME=batman-adv
+PACKAGE_VERSION={{ pillar.batman_adv.version }}
+
+DEST_MODULE_LOCATION=/extra
+BUILT_MODULE_NAME=batman-adv
+BUILT_MODULE_LOCATION=net/batman-adv/
+
+MAKE="make KERNELPATH=${kernel_source_dir}"
+CLEAN="make KERNELPATH=${kernel_source_dir} clean"
+AUTOINSTALL="yes"

freifunk-common/files/batman/install-dkms.sh

+#!/bin/bash
+set -e
+
+# clean up
+for version in $(/usr/sbin/dkms status | egrep "^batman-adv," | cut -f 2 -d ' ' | sed 's/,//' | sort -u); do
+    dkms remove batman-adv/"$version" --all
+done
+
+# install current version
+dkms add -m batman-adv -v {{ pillar.batman_adv.version }}
+dkms autoinstall
+
+# test
+if ! dkms status | egrep "batman-adv, {{ pillar.batman_adv.version }}, .*, x86_64: installed"; then
+    dkms status
+    exit 1
+fi

freifunk-common/files/journald.conf

+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+#
+# See journald.conf(5) for details
+
+[Journal]
+Storage=persistent
+#Compress=yes
+#Seal=yes
+#SplitMode=uid
+#SyncIntervalSec=5m
+#RateLimitInterval=30s
+#RateLimitBurst=1000
+SystemMaxUse=500M
+#SystemKeepFree=
+#SystemMaxFileSize=
+RuntimeMaxUse=100M
+#RuntimeKeepFree=
+#RuntimeMaxFileSize=
+MaxRetentionSec=1week
+#MaxFileSec=1month
+#ForwardToSyslog=yes
+#ForwardToKMsg=no
+#ForwardToConsole=no
+#ForwardToWall=yes
+#TTYPath=/dev/console
+MaxLevelStore=notice
+#MaxLevelSyslog=debug
+#MaxLevelKMsg=notice
+#MaxLevelConsole=info
+#MaxLevelWall=emerg

freifunk-common/files/network_interfaces/000-freifunk

+#
+# FREIFUNK INTERFACES
+#
+{% for domain in salt.ffsaar.domains() %}
+auto {{ domain.name }}BR
+iface {{ domain.name }}BR inet6 manual
+    ### Bring up interface ###
+    pre-up modprobe dummy
+    # create and configure bridge
+    pre-up ip link add $IFACE address {{ domain.intranet_mac() }} type bridge
+    pre-up ip addr add {{ domain.host() }}/{{ domain.net().netmask }} dev $IFACE
+    pre-up ip addr add {{ domain.host_6() }}/{{ domain.net_6().prefixlen }} dev $IFACE
+{% if salt.ffsaar.is_gw() %}
+    pre-up ip addr add {{ domain.ffrl_host_6() }}/{{ domain.ffrl_net_6().prefixlen }} dev $IFACE
+{% endif %}
+{% if pillar.servername == "mgmt" and domain.name == "saar" %}
+    # mgmt: IP address for TP-LINK router flashing (only available in the saar domain to avoid
+    # duplicate IP addresses)
+    pre-up ip addr add 192.168.0.66/24 dev $IFACE
+{% endif %}
+{% if salt.ffsaar.is_gw() and domain.name == "saar" %}
+    # GW compatibility addresses (FIXME: remove)
+    pre-up ip addr add fd4e:f2d7:88d2:ffff::{{ 1 + salt.ffsaar.gw_num() }}/64 dev $IFACE
+    pre-up ip addr add 10.24.192.{{ 1 + salt.ffsaar.gw_num() }}/19 dev $IFACE
+{% endif %}
+    # We need a dummy interface to set an MTU
+    pre-up ip link add $IFACE-mtu type dummy
+    pre-up ip link set $IFACE-mtu mtu {{ pillar.mtu_clients }} master $IFACE
+{% if salt.ffsaar.is_gw() %}
+    # use policy routing
+    pre-up ip -4 rule add from {{ domain.net() }} table freifunk priority 16
+    pre-up ip -4 rule add iif $IFACE unreachable priority 32
+    pre-up ip -6 rule add from {{ domain.net_6() }} table freifunk priority 16
+    pre-up ip -6 rule add from {{ domain.ffrl_net_6() }} table freifunk priority 16
+    pre-up ip -6 rule add iif $IFACE unreachable priority 32
+{% endif %}
+    # Make sure we gather multicast data for batman to use
+    post-up echo 1024 > /sys/class/net/$IFACE/bridge/hash_max
+    post-up echo 1 > /sys/class/net/$IFACE/bridge/multicast_snooping
+    ### Shutdown interface ###
+    post-down ip link delete $IFACE type bridge
+
+allow-hotplug {{ domain.name }}BAT
+iface {{ domain.name }}BAT inet6 manual
+    pre-up         batctl meshif $IFACE it 10000
+    pre-up         batctl meshif $IFACE f 0
+    post-up         /sbin/brctl addif {{ domain.name }}BR $IFACE
+    # multicast_router = 2 disables multicast router learning and always forwards all multicast traffic to that bridge port
+    # the gluon nodes are multicast routers so that's what we want
+    post-up         echo 2 > /sys/class/net/$IFACE/brport/multicast_router
+    post-up         batctl meshif $IFACE hop_penalty 30
+    post-up         batctl meshif $IFACE gw off
+    pre-down        /sbin/brctl delif {{ domain.name }}BR $IFACE || true
+
+{% if salt.ffsaar.is_gw() %}
+# This is a gateway, it has a VPN bridge.
+# The VPN bridge collects links to all l2tp nodes.
+# For some reason, things don't work without this bridge.
+auto {{ domain.name }}VPN
+iface {{ domain.name }}VPN inet6 manual
+    ## Bring up interface
+    pre-up ip link add $IFACE address {{ domain.batman_mac() }} type bridge
+    pre-up ip link set $IFACE promisc on # given in tunneldigger doku
+    up ip link set dev $IFACE up
+    post-up ebtables -A FORWARD --logical-in $IFACE -j DROP # don't do any switching
+    post-up batctl meshif {{ domain.name }}BAT if add $IFACE
+    ## Shutdown interface
+    pre-down batctl meshif {{ domain.name }}BAT if del $IFACE
+    pre-down ebtables -D FORWARD --logical-in $IFACE -j DROP
+    down ip link set dev $IFACE down
+    post-down ip link delete $IFACE type bridge
+{% endif %}
+{% endfor %}

freifunk-common/files/network_interfaces/inter_gw

+{% for sv in pillar.servers %}
+{% if sv.name != pillar.servername %}
+{% for domain in pillar.domains %}
+
+auto {{ domain.name }}-tun-{{ sv.name }}
+iface {{ domain.name }}-tun-{{ sv.name }} inet manual
+    pre-up         ip link add $IFACE type gretap local {{ salt.ffsaar.server_pubip() }} remote {{ sv.ip }} key {{ domain.inter_gw_gre_key }} ttl 255
+    up ip link set dev $IFACE up
+    post-up ip link set $IFACE mtu {{ pillar.mtu_tunnel }}
+    post-up batctl meshif {{ domain.name }}BAT if add $IFACE
+    pre-down batctl meshif {{ domain.name }}BAT if del $IFACE
+    post-down ip link del $IFACE type gretap
+
+{% endfor %}
+{% endif %}
+{% endfor %}

freifunk-common/files/ntp.conf

+# /etc/ntp.conf, configuration for ntpd; see ntp.conf(5) for help
+
+driftfile /var/lib/ntp/ntp.drift
+
+# Leap seconds definition provided by tzdata
+leapfile /usr/share/zoneinfo/leap-seconds.list
+
+# Enable this if you want statistics to be logged.
+#statsdir /var/log/ntpstats/
+
+statistics loopstats peerstats clockstats
+filegen loopstats file loopstats type day enable
+filegen peerstats file peerstats type day enable
+filegen clockstats file clockstats type day enable
+
+
+# You do need to talk to an NTP server or two (or three).
+#server ntp.your-provider.example
+
+# pool.ntp.org maps to about 1000 low-stratum NTP servers.  Your server will
+# pick a different set every time it starts up.  Please consider joining the
+# pool: <http://www.pool.ntp.org/join.html>
+pool 0.debian.pool.ntp.org iburst
+pool 1.debian.pool.ntp.org iburst
+pool 2.debian.pool.ntp.org iburst
+pool 3.debian.pool.ntp.org iburst
+
+
+# Access control configuration; see /usr/share/doc/ntp-doc/html/accopt.html for
+# details.  The web page <http://support.ntp.org/bin/view/Support/AccessRestrictions>
+# might also be helpful.
+#
+# Note that "restrict" applies to both servers and clients, so a configuration
+# that might be intended to block requests from certain clients could also end
+# up blocking replies from your own upstream servers.
+
+# By default, exchange time with everybody, but don't allow configuration.
+restrict -4 default kod notrap nomodify nopeer noquery limited
+restrict -6 default kod notrap nomodify nopeer noquery limited
+
+# Local users may interrogate the ntp server more closely.
+restrict 127.0.0.1
+restrict ::1
+
+# Needed for adding pool entries
+restrict source notrap nomodify noquery
+
+# Clients from this (example!) subnet have unlimited access, but only if
+# cryptographically authenticated.
+#restrict 192.168.123.0 mask 255.255.255.0 notrust
+
+
+# If you want to provide time to your local subnet, change the next line.
+# (Again, the address is an example only.)
+#broadcast 192.168.123.255
+
+# If you want to listen to time broadcasts on your local subnet, de-comment the
+# next lines.  Please do this only if you trust everybody on the network!
+#disable auth
+#broadcastclient

freifunk-common/files/respondd/config.json

+{
+  "mcast_iface": "saarBR",
+  "mcast_group": "ff05::2:1001",
+  "bat_iface": [
+{%- for domain in pillar.domains %}
+      "{{ domain.name }}BAT"
+      {%- if not loop.last %},{% endif -%}
+{%- endfor %}
+  ],
+  "site_code": "ffsaar"
+}

freifunk-common/files/respondd/node-respondd.service

+[Unit]
+Description=The node-respondd server provides data about this node to the local network.
+Requires=sys-devices-virtual-net-saarBR.device
+After=sys-devices-virtual-net-saarBR.device
+After=network-online.target
+
+[Service]
+Type=simple
+WorkingDirectory=/opt/node-respondd
+ExecStart=/opt/node-respondd/respondd
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

freifunk-common/files/scripts/bat-find-client

+#!/usr/bin/env python3
+import sys, re, subprocess
+
+if len(sys.argv) < 2:
+    print("Usage: {} <MAC address as shown in hopglass> <domain name?>".format(sys.argv[0]))
+    sys.exit(1)
+target = sys.argv[1]
+domain = sys.argv[2] if 2 < len(sys.argv) else "saar"
+print("Trying to find {} in the {} BAT network.".format(target, domain))
+
+# First thing: ask batman for the "originator" for the address.
+originator = subprocess.check_output(["batctl", "meshif", domain+"BAT", "t", target]).decode('utf-8').strip()
+print("Originator is {}.".format(originator))
+
+# Next, look up this originator in the originator table to determine the next hop.
+# This tells us the (outer) MAC address that Batman will tell Linux to send packages for this MAC to.
+nextMAC = None
+nextIface = None
+for line in subprocess.check_output(["batctl", "meshif", domain+"BAT", "o"]).decode("utf-8").split('\n'):
+    # look for the line with the *, I think that's the "best" one or so
+    m = re.match(r'^\s+\*\s+([0-9a-f:]+)\s+([0-9.]+)s\s+\(\s*([0-9]+)\s*\)\s+([0-9a-f:]+)\s+\[\s*([a-zA-Z0-9-]+)\s*\]', line) # groups: originator, last-seen, throughput, next best hop, outgoing iface
+    if m is None: continue # spurious line, e.g. the header
+    if m.group(1) == originator:
+        nextMAC = m.group(4)
+        nextIface = m.group(5)
+if nextMAC is None or nextIface is None:
+    print("ERROR: Could not find originator in originator table.")
+    sys.exit(1)
+print("Next hop is towards {} via {}.".format(nextMAC, nextIface))
+
+if nextIface != domain+"VPN":
+    print("This client is connected via some other GW, cannot tell more from here.")
+    sys.exit(0)
+
+# Next, ask the bridge for the port that this next hop is connected to
+port = None
+for line in subprocess.check_output(["brctl", "showmacs", domain+"VPN"]).decode("utf-8").split('\n'):
+    m =	re.match(r'^\s*([0-9]+)\s+([0-9a-f:]+)\s+(yes|no)\s+([0-9.]+)\s*$', line) # groups: port number, MAC, "local?", ageing
+    if m is None: continue # spurious line, e.g. the header
+    if m.group(2) == nextMAC:
+        assert m.group(3) == "no"
+        port = m.group(1)
+        break
+if port is None:
+    print("ERROR: Could not find originator in bridge MAC table.")
+    sys.exit(1)
+print("Bridge port is {}.".format(port))
+
+# Finally, figure out the device corresponding to this bridge port
+device = None
+for line in subprocess.check_output(["brctl", "showstp", domain+"VPN"]).decode("utf-8").split('\n'):
+    m = re.match(r'^([a-zA-Z0-9-_]+)\s+\(([0-9]+)\)$', line) # groups: interface name, port number
+    if m is None: continue # spurious line, e.g. the header
+    if m.group(2) == port:
+        device = m.group(1)
+        break
+if device is None:
+    print("ERROR: Could not find device for bridge port.")
+    sys.exit(1)
+print("Device name is {}.".format(device))
+
+# Maybe we can even find sth. about this port in the logs?
+print()
+print("Syslog entries concerning tunnel {}:".format(device))
+for line in subprocess.check_output(["journalctl", "-b", "0", "-u", "tunneldigger"]).decode("utf-8").split('\n'):
+    if line.endswith(device):
+        print(line)

freifunk-common/files/scripts/hard-reboot

+#!/bin/bash
+echo s | sudo tee /proc/sysrq-trigger
+sleep 1
+echo u | sudo tee /proc/sysrq-trigger
+sleep 1
+echo b | sudo tee /proc/sysrq-trigger

freifunk-common/files/scripts/more-logging

+#!/bin/sh
+
+echo "Enabling more logging.  Be sure to re-deploy salt to get us back to our usual logging settings!"
+sudo sed -i 's,MaxLevelStore,#MaxLevelStore,' /etc/systemd/journald.conf
+sudo systemctl restart systemd-journald

freifunk-common/files/scripts/restart-gre

+#!/bin/sh
+set -e
+{% if salt.ffsaar.is_gw() and pillar.use_ffrl %}