Jonas Nick
34a18f92ee
The `netns-exec` wrapper was created with `group = ""`, which makes the
`security.wrappers` activation run `chown ${allowedUser}:`. GNU chown
resolves the trailing colon to the owner's login group, and because the
operator is defined with `isNormalUser = true`, that group is the shared
group `users`. Together with mode 550, this made the wrapper executable
by every normal user on the host instead of only by `allowedUser`.
The wrapper carries `cap_sys_admin=ep` and the NixOS security wrapper
performs no owner check, so any normal user could enter `nb-joinmarket`,
the only netns that `netns-exec` permits. This exposes joinmarketd's
unauthenticated control port on 127.0.0.1 inside that netns and allows
sending packets from the joinmarket netns address. This can only ever become a
problem if there's a second, normal user on the host.
Use mode 500 so that only `allowedUser` can execute the wrapper. Also
set the group to `root`, so that `users` doesn't silently become an
authorized group again if the mode is ever loosened.
The netns-isolation test asserted this property with
`runuser -u clightning -- netns-exec nb-bitcoind ip a`, which fails
regardless of the file mode, because `clightning` is a system user with
its own group and `nb-bitcoind` is not in netns-exec's allowlist.
Replace it with a check that a normal user is denied by the exec
permissions, and assert the reason for each failure instead of only the
exit status.
367 lines
11 KiB
Nix
367 lines
11 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
|
|
with lib;
|
|
let
|
|
options.nix-bitcoin.netns-isolation = {
|
|
enable = mkEnableOption "netns isolation";
|
|
|
|
addressblock = mkOption {
|
|
type = types.ints.u8;
|
|
default = 1;
|
|
description = ''
|
|
The address block N in 169.254.N.0/24, used as the prefix for netns addresses.
|
|
'';
|
|
};
|
|
|
|
services = mkOption {
|
|
default = {};
|
|
type = types.attrsOf (types.submodule {
|
|
options = {
|
|
id = mkOption {
|
|
# TODO: Assert uniqueness
|
|
type = types.ints.between 11 255;
|
|
description = ''
|
|
id for the netns, used for the IP address host part and
|
|
for naming the interfaces. Must be unique. Must be greater than 10.
|
|
'';
|
|
};
|
|
connections = mkOption {
|
|
type = with types; listOf str;
|
|
default = [];
|
|
};
|
|
};
|
|
});
|
|
};
|
|
|
|
allowedUser = mkOption {
|
|
type = types.str;
|
|
description = ''
|
|
User that is allowed to execute commands in the service network namespaces.
|
|
'';
|
|
default = config.nix-bitcoin.operator.name;
|
|
};
|
|
|
|
netns = mkOption {
|
|
readOnly = true;
|
|
default = netns;
|
|
description = "Exposes netns parameters.";
|
|
};
|
|
|
|
bridgeIp = mkOption {
|
|
readOnly = true;
|
|
default = bridgeIp;
|
|
description = "IP of the netns bridge interface.";
|
|
};
|
|
};
|
|
|
|
cfg = config.nix-bitcoin.netns-isolation;
|
|
|
|
netns = builtins.mapAttrs (n: v: {
|
|
inherit (v) id;
|
|
address = "169.254.${toString cfg.addressblock}.${toString v.id}";
|
|
availableNetns = availableNetns.${n};
|
|
netnsName = "nb-${n}";
|
|
}) enabledServices;
|
|
|
|
# Symmetric netns connection matrix
|
|
# if clightning.connections = [ "bitcoind" ]; then
|
|
# availableNetns.bitcoind = [ "clighting" ];
|
|
# and
|
|
# availableNetns.clighting = [ "bitcoind" ];
|
|
#
|
|
# TODO-EXTERNAL:
|
|
# Although negligible for our purposes, this calculation's runtime
|
|
# is in the order of (number of connections * number of services),
|
|
# because attrsets and lists are fully copied on each update with '//' or '++'.
|
|
# This can only be improved with an update in the nix language.
|
|
#
|
|
availableNetns = let
|
|
# base = { clightning = [ "bitcoind" ]; ... }
|
|
base = builtins.mapAttrs (n: v:
|
|
builtins.filter isEnabled v.connections
|
|
) enabledServices;
|
|
in
|
|
foldl (xs: s1:
|
|
foldl (xs: s2:
|
|
xs // { "${s2}" = xs.${s2} ++ [ s1 ]; }
|
|
) xs cfg.services.${s1}.connections
|
|
) base (builtins.attrNames base);
|
|
|
|
enabledServices = filterAttrs (n: v: isEnabled n) cfg.services;
|
|
isEnabled = x: config.services.${x}.enable;
|
|
|
|
ip = "${pkgs.iproute2}/bin/ip";
|
|
iptables = "${config.networking.firewall.package}/bin/iptables";
|
|
|
|
bridgeIp = "169.254.${toString cfg.addressblock}.10";
|
|
|
|
mkCliExec = service: "exec netns-exec ${netns.${service}.netnsName}";
|
|
in {
|
|
inherit options;
|
|
|
|
config = mkIf cfg.enable (mkMerge [
|
|
|
|
# Base infrastructure
|
|
{
|
|
networking.dhcpcd.denyInterfaces = [ "nb-br" "nb-veth*" ];
|
|
services.tor.client.socksListenAddress = {
|
|
addr = bridgeIp;
|
|
# Default NixOS values. These must be repeated when redefining this option.
|
|
port = 9050;
|
|
IsolateDestAddr = true;
|
|
};
|
|
services.i2pd.proto.sam.address = bridgeIp;
|
|
networking.firewall.interfaces.nb-br.allowedTCPPorts = [
|
|
config.services.tor.client.socksListenAddress.port
|
|
config.services.i2pd.proto.sam.port
|
|
];
|
|
boot.kernel.sysctl."net.ipv4.ip_forward" = true;
|
|
|
|
security.wrappers.netns-exec = {
|
|
source = config.nix-bitcoin.pkgs.netns-exec;
|
|
capabilities = "cap_sys_admin=ep";
|
|
owner = cfg.allowedUser;
|
|
# Don't authorize the group of `owner`. For normal users, this group is
|
|
# `users`, which is shared by all normal users.
|
|
group = "root";
|
|
permissions = "500";
|
|
};
|
|
|
|
systemd.services = {
|
|
# Due to a NixOS bug we can't currently use option `networking.bridges` to
|
|
# setup the bridge while `networking.useDHCP` is enabled.
|
|
nb-netns-bridge = {
|
|
description = "nix-bitcoin netns bridge";
|
|
wantedBy = [ "network-setup.service" ];
|
|
partOf = [ "network-setup.service" ];
|
|
before = [ "network-setup.service" ];
|
|
after = [ "network-pre.target" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
};
|
|
script = ''
|
|
${ip} link add name nb-br type bridge
|
|
${ip} link set nb-br up
|
|
${ip} addr add ${bridgeIp}/24 brd + dev nb-br
|
|
${iptables} -w -t nat -A POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
|
|
'';
|
|
preStop = ''
|
|
${iptables} -w -t nat -D POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
|
|
${ip} link del nb-br
|
|
'';
|
|
};
|
|
|
|
} //
|
|
(let
|
|
makeNetnsServices = n: v: let
|
|
veth = "nb-veth-${toString v.id}";
|
|
peer = "nb-veth-br-${toString v.id}";
|
|
inherit (v) netnsName;
|
|
nsenter = "${pkgs.util-linux}/bin/nsenter";
|
|
allowedNetnsAddresses = map (available: netns.${available}.address) v.availableNetns;
|
|
allowedAddresses = concatStringsSep ","
|
|
([ "127.0.0.1,${bridgeIp},${v.address}" ] ++ allowedNetnsAddresses);
|
|
|
|
setup = ''
|
|
${ip} netns add ${netnsName}
|
|
${ip} link add ${veth} type veth peer name ${peer}
|
|
${ip} link set ${veth} netns ${netnsName}
|
|
# The peer link is never used directly, so don't auto-assign an IPv6 address
|
|
echo 1 > /proc/sys/net/ipv6/conf/${peer}/disable_ipv6
|
|
${ip} link set ${peer} up
|
|
${ip} link set ${peer} master nb-br
|
|
exec ${nsenter} --net=/run/netns/${netnsName} ${script "in-netns" setupInNetns}
|
|
'';
|
|
|
|
setupInNetns = ''
|
|
${ip} link set lo up
|
|
${ip} addr add ${v.address}/24 dev ${veth}
|
|
${ip} link set ${veth} up
|
|
${ip} route add default via ${bridgeIp}
|
|
|
|
${iptables} -w -P INPUT DROP
|
|
# allow return traffic to outgoing connections initiated by the service itself
|
|
${iptables} -w -A INPUT -m conntrack --ctstate ESTABLISHED -j ACCEPT
|
|
${iptables} -w -A INPUT -s ${allowedAddresses} -j ACCEPT
|
|
'' + optionalString (config.services.${n}.tor.enforce or false) ''
|
|
${iptables} -w -P OUTPUT DROP
|
|
${iptables} -w -A OUTPUT -d ${allowedAddresses} -j ACCEPT
|
|
'';
|
|
script = name: src: pkgs.writers.writeDash name ''
|
|
set -e
|
|
${src}
|
|
'';
|
|
in {
|
|
"${n}".serviceConfig.NetworkNamespacePath = "/var/run/netns/${netnsName}";
|
|
|
|
"netns-${n}" = rec {
|
|
requires = [ "nb-netns-bridge.service" ];
|
|
after = [ "nb-netns-bridge.service" ];
|
|
requiredBy = [ "${n}.service" ];
|
|
before = requiredBy;
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = script "setup" setup;
|
|
};
|
|
# Link deletion is implicit in netns deletion, but it sometimes only happens
|
|
# after `netns delete` finishes. Add an extra `link del` to ensure that
|
|
# the link is deleted before the service stops, which is needed for service
|
|
# restart to succeed.
|
|
preStop = ''
|
|
${ip} netns delete ${netnsName}
|
|
${ip} link del ${peer} 2> /dev/null || true
|
|
'';
|
|
|
|
};
|
|
};
|
|
in
|
|
foldl (services: n:
|
|
services // (makeNetnsServices n netns.${n})
|
|
) {} (builtins.attrNames netns)
|
|
);
|
|
}
|
|
|
|
# Service-specific config
|
|
{
|
|
nix-bitcoin.netns-isolation.services = {
|
|
bitcoind = {
|
|
id = 12;
|
|
};
|
|
clightning = {
|
|
id = 13;
|
|
connections = [ "bitcoind" ];
|
|
};
|
|
lnd = {
|
|
id = 14;
|
|
connections = [ "bitcoind" ];
|
|
};
|
|
liquidd = {
|
|
id = 15;
|
|
connections = [ "bitcoind" ];
|
|
};
|
|
electrs = {
|
|
id = 16;
|
|
connections = [ "bitcoind" ];
|
|
};
|
|
nginx = {
|
|
id = 21;
|
|
};
|
|
lightning-loop = {
|
|
id = 22;
|
|
connections = [ "lnd" ];
|
|
};
|
|
nbxplorer = {
|
|
id = 23;
|
|
connections = [ "bitcoind" ]
|
|
++ optional config.services.btcpayserver.lbtc "liquidd";
|
|
};
|
|
btcpayserver = {
|
|
id = 24;
|
|
connections = [ "nbxplorer" ]
|
|
++ optional (config.services.btcpayserver.lightningBackend == "lnd") "lnd"
|
|
++ optional config.services.btcpayserver.lbtc "liquidd";
|
|
# communicates with clightning over rpc socket
|
|
};
|
|
joinmarket = {
|
|
id = 25;
|
|
connections = [ "bitcoind" ];
|
|
};
|
|
joinmarket-ob-watcher = {
|
|
id = 26;
|
|
connections = [ "bitcoind" ];
|
|
};
|
|
lightning-pool = {
|
|
id = 27;
|
|
connections = [ "lnd" ];
|
|
};
|
|
charge-lnd = {
|
|
id = 28;
|
|
connections = [ "lnd" "electrs" ];
|
|
};
|
|
rtl = {
|
|
id = 29;
|
|
connections = let
|
|
nodes = config.services.rtl.nodes;
|
|
in
|
|
optional nodes.lnd.enable "lnd" ++
|
|
optional (nodes.lnd.enable && nodes.lnd.loop) "lightning-loop" ++
|
|
optional nodes.clightning.enable "clightning";
|
|
};
|
|
clightning-rest = {
|
|
id = 30;
|
|
};
|
|
fulcrum = {
|
|
id = 31;
|
|
connections = [ "bitcoind" ];
|
|
};
|
|
mempool = {
|
|
id = 32;
|
|
connections = [
|
|
"bitcoind"
|
|
"nginx"
|
|
(if (config.services.mempool.electrumServer == "electrs") then "electrs" else "fulcrum")
|
|
];
|
|
};
|
|
};
|
|
|
|
services.bitcoind = {
|
|
address = netns.bitcoind.address;
|
|
rpc.address = netns.bitcoind.address;
|
|
rpc.allowip = [
|
|
bridgeIp # For operator user
|
|
netns.bitcoind.address
|
|
] ++ map (n: netns.${n}.address) netns.bitcoind.availableNetns;
|
|
};
|
|
|
|
services.clightning.address = netns.clightning.address;
|
|
services.clightning.plugins.clnrest.address = netns.clightning.address;
|
|
|
|
services.lnd = {
|
|
address = netns.lnd.address;
|
|
rpcAddress = netns.lnd.address;
|
|
restAddress = netns.lnd.address;
|
|
};
|
|
|
|
services.liquidd = {
|
|
address = netns.liquidd.address;
|
|
rpc.address = netns.liquidd.address;
|
|
rpcallowip = [
|
|
bridgeIp # For operator user
|
|
netns.liquidd.address
|
|
] ++ map (n: netns.${n}.address) netns.liquidd.availableNetns;
|
|
};
|
|
|
|
services.electrs.address = netns.electrs.address;
|
|
|
|
services.fulcrum.address = netns.fulcrum.address;
|
|
|
|
services.lightning-loop.rpcAddress = netns.lightning-loop.address;
|
|
|
|
services.nbxplorer.address = netns.nbxplorer.address;
|
|
services.btcpayserver.address = netns.btcpayserver.address;
|
|
|
|
services.joinmarket = mkIf config.services.joinmarket.enable {
|
|
payjoinAddress = netns.joinmarket.address;
|
|
messagingAddress = netns.joinmarket.address;
|
|
cliExec = mkCliExec "joinmarket";
|
|
};
|
|
systemd.services.joinmarket-yieldgenerator = mkIf config.services.joinmarket.yieldgenerator.enable {
|
|
serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-joinmarket";
|
|
};
|
|
|
|
services.joinmarket-ob-watcher.address = netns.joinmarket-ob-watcher.address;
|
|
|
|
services.lightning-pool.rpcAddress = netns.lightning-pool.address;
|
|
|
|
services.rtl.address = netns.rtl.address;
|
|
|
|
services.clightning-rest.address = netns.clightning-rest.address;
|
|
|
|
services.mempool.address = netns.mempool.address;
|
|
services.mempool.frontend.address = netns.nginx.address;
|
|
}
|
|
]);
|
|
}
|