Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support UDP/TCP port fowarding to a host without setting up a tun #1179

Open
wants to merge 35 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
f119439
performance impr.: avoid repeated allocation of "lastTick" on heap
cre4ture Aug 11, 2024
6eaf418
support unsafe_routes for use with port_forwarding and user_tun
cre4ture Aug 5, 2024
2388e2c
performance 3: avoid repeated allocation of []byte{1} on heap
cre4ture Aug 11, 2024
aa41526
performance4: use buffer.view based channels instead of pipe
cre4ture Aug 11, 2024
c245a30
Add script for speedtesting port forwarding
akernet Jul 21, 2024
ed937ab
Revert "Add script for speedtesting port forwarding"
cre4ture Aug 11, 2024
48745eb
modified and extended speedtest
cre4ture Aug 11, 2024
c516383
add automated functional tests for port forwarding
cre4ture Aug 11, 2024
067410f
TCP/UDP port fwd. for disabled tun with config reload support
cre4ture Aug 11, 2024
ea77cde
fix fmt
cre4ture Aug 11, 2024
ac016c9
try to fix instability of the service level tests
cre4ture Aug 12, 2024
a678cff
lets see if randomization of the port helps
cre4ture Aug 12, 2024
9d60a1b
avoid panic due to writing to closed channel
cre4ture Aug 12, 2024
bd55362
extend validity time range to avoid race conditions in CI
cre4ture Aug 13, 2024
a9b0b1d
ensure that node certs lifetime doesn't outlife ca cert lifetime
cre4ture Aug 14, 2024
d22fe21
consider injected name for certificate configuration
cre4ture Aug 14, 2024
670b3ab
try to unique node names in tests
cre4ture Aug 14, 2024
6d03850
add name of the test to the certificate
cre4ture Aug 15, 2024
3fd775b
add logging prefix to differentiate the output in a test with 2 services
cre4ture Aug 15, 2024
99b11b3
tests: check for clean service shutdown and improve logging
cre4ture Aug 16, 2024
ca43832
try to make it more stable by using channels and waitgroups
cre4ture Aug 17, 2024
12a0dd8
improve stopping logic for UserDevice
cre4ture Aug 17, 2024
b1ea9f5
improving test code to get stability improved - still fails with stress
cre4ture Aug 19, 2024
352f74f
Merge remote-tracking branch 'origin/master' into feature/try_with_gv…
cre4ture Aug 27, 2024
84d1a26
fix issue with survival of nebula service from previous testrun
cre4ture Sep 14, 2024
ad2dbdc
require instead of assert; fix missing close connectons in one test
cre4ture Sep 15, 2024
fa7d120
service: fix missing destruction of ipstack
cre4ture Sep 15, 2024
e6bcba2
add comment to "unsafe_routes" initialisation
cre4ture Sep 16, 2024
f908c10
add comment to performance improvement in user-tun
cre4ture Sep 16, 2024
ba8a037
add missing error handling when calling fwd factorie functions
cre4ture Sep 16, 2024
ba7880a
remove all sleeps in tests - no need for them
cre4ture Sep 16, 2024
39d8332
nitpick: use atomic bool instead of bool for usynced thread access
cre4ture Sep 16, 2024
cd510b3
fix race condition where "CloseAndZero" is executed while still used
cre4ture Sep 16, 2024
c2e4dd9
fix the closing of the linux udp reading loop using Shutdown
cre4ture Sep 17, 2024
03630fb
Merge remote-tracking branch 'origin/master' into feature/try_with_gv…
cre4ture Dec 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 54 additions & 8 deletions cmd/nebula/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@ import (
"flag"
"fmt"
"os"
"os/signal"
"syscall"

"github.com/sirupsen/logrus"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/port_forwarder"
"github.com/slackhq/nebula/service"
"github.com/slackhq/nebula/util"
)

Expand Down Expand Up @@ -52,16 +56,58 @@ func main() {
os.Exit(1)
}

ctrl, err := nebula.Main(c, *configTest, Build, l, nil)
if err != nil {
util.LogWithContextIfNeeded("Failed to start", err, l)
os.Exit(1)
fwd_list := port_forwarder.NewPortForwardingList()
disabled_tun := c.GetBool("tun.disabled", false)
activate_service_anyway := c.GetBool("port_forwarding.enable_without_rules", false)
if disabled_tun {
port_forwarder.ParseConfig(l, c, fwd_list)
}

if !*configTest {
ctrl.Start()
notifyReady(l)
ctrl.ShutdownBlock()
if !*configTest && disabled_tun && (activate_service_anyway || !fwd_list.IsEmpty()) {
l.Infof("Configuring user-tun instead of disabled-tun as port forwarding is configured")

service, err := service.New(c, l)
if err != nil {
util.LogWithContextIfNeeded("Failed to create service", err, l)
os.Exit(1)
}

// initialize port forwarding:
pf_service, err := port_forwarder.ConstructFromInitialFwdList(service, l, &fwd_list)
if err != nil {
util.LogWithContextIfNeeded("Failed to start", err, l)
os.Exit(1)
}

c.RegisterReloadCallback(func(c *config.C) {
pf_service.ReloadConfigAndApplyChanges(c)
})

pf_service.Activate()

// wait for termination request
signalChannel := make(chan os.Signal, 1)
signal.Notify(signalChannel, syscall.SIGINT, syscall.SIGTERM)
fmt.Println("Running, press ctrl+c to shutdown...")
<-signalChannel

// shutdown:
service.CloseAndWait()

} else {

l.Info("Configuring for disabled or kernel tun. no port forwarding provided")
ctrl, err := nebula.Main(c, *configTest, Build, l, nil)
if err != nil {
util.LogWithContextIfNeeded("Failed to start", err, l)
os.Exit(1)
}

if !*configTest {
ctrl.Start()
notifyReady(l)
ctrl.ShutdownBlock()
}
}

os.Exit(0)
Expand Down
9 changes: 7 additions & 2 deletions connection_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ const (
sendTestPacket trafficDecision = 6
)

// The data written into this variable is never used.
// Its there to avoid a fresh dynamic memory allocation of 1 byte
// for each time its used.
var BYTE_SLICE_ONE []byte = []byte{1}

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest moving this to a separate PR since it affects normal configs too

type connectionManager struct {
in map[uint32]struct{}
inLock *sync.RWMutex
Expand Down Expand Up @@ -463,12 +468,12 @@ func (n *connectionManager) sendPunch(hostinfo *HostInfo) {
if n.punchy.GetTargetEverything() {
hostinfo.remotes.ForEach(n.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
n.metricsTxPunchy.Inc(1)
n.intf.outside.WriteTo([]byte{1}, addr)
n.intf.outside.WriteTo(BYTE_SLICE_ONE, addr)
})

} else if hostinfo.remote.IsValid() {
n.metricsTxPunchy.Inc(1)
n.intf.outside.WriteTo([]byte{1}, hostinfo.remote)
n.intf.outside.WriteTo(BYTE_SLICE_ONE, hostinfo.remote)
}
}

Expand Down
3 changes: 3 additions & 0 deletions e2e/forwarding/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
*.out
*.crt
*.key
27 changes: 27 additions & 0 deletions e2e/forwarding/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Userspace port forwarding
A simple speedtest for userspace port forwarding that can run without root access.

## A side
Nebula running at port 10000, forwarding inbound TCP connections on port 5201 to 127.0.0.1:15001.

## B side
Nebula running at port 10001, forwarding outbound TCP connections from 127.0.0.1:15002 to port 5201 of the A side.

## Speedtest

┌──────────────────────┐:10001 :10002┌──────────────────────┐
│ Nebula A side ├─────────────────┤ Nebula B side │
│ │ │ │
│ 192.168.100.1 │ TCP 5201 │ 192.168.100.2 │
│ ┌───────────┼─────────────────┼──────────┐ │
│ │ ├─────────────────┤ │ │
└──────────▼───────────┘ └──────────▲───────────┘
│ │ 127.0.0.1:15002
│ │
┌──────────▼───────────┐ ┌──────────┴───────────┐
│ │ │ │
│ │ │ │
│ iperf3 -s -p 15001 │ │ iperf3 -c -p 15001 │
│ │ │ │
│ │ │ │
└──────────────────────┘ └──────────────────────┘
35 changes: 35 additions & 0 deletions e2e/forwarding/a_config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
pki:
ca: ca.crt
cert: a.crt
key: a.key

static_host_map:
"192.168.100.2": ["127.0.0.1:10002"]

logging:
level: info

listen:
host: 127.0.0.1
port: 10001

port_forwarding:
enable_without_rules: true
inbound:
- listen_port: 5201
dial_address: "127.0.0.1:15001"
protocols: [tcp, udp]

tun:
disabled: true
mtu: 1300

firewall:
outbound:
- port: any
proto: udp
host: any
inbound:
- port: 5201
proto: any
host: any
34 changes: 34 additions & 0 deletions e2e/forwarding/b_config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
pki:
ca: ca.crt
cert: b.crt
key: b.key

static_host_map:
"192.168.100.1": ["127.0.0.1:10001"]

logging:
level: info

listen:
host: 127.0.0.1
port: 10002

port_forwarding:
enable_without_rules: true
outbound:
- listen_address: "127.0.0.1:15002"
dial_address: "192.168.100.1:5201"
protocols: [tcp, udp]

tun:
disabled: true
mtu: 1300

firewall:
outbound:
- port: any
proto: udp
host: any
- port: 5201
proto: any
host: any
5 changes: 5 additions & 0 deletions e2e/forwarding/generate_certificates.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash

../../nebula-cert ca -name "E2E test CA"
../../nebula-cert sign -name "A" -ip "192.168.100.1/24" -out-crt a.crt -out-key a.key
../../nebula-cert sign -name "B" -ip "192.168.100.2/24" -out-crt b.crt -out-key b.key
36 changes: 36 additions & 0 deletions e2e/forwarding/speedtest.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/bin/bash

cd "$(dirname "$0")"

if ! test -f ca.key; then
echo "Generating new test certificates"
./generate_certificates.sh
fi

../../nebula -config "$(pwd)/a_config.yml" &>a.out &
A_PID=$!
../../nebula -config "$(pwd)/b_config.yml" &>b.out &
B_PID=$!

iperf3 -s -p 15001 &
IPERF_SERVER_PID=$!

sleep 1
iperf3 -c 127.0.0.1 -p 15002 -P 10 "$@"

# Cleanup
kill $IPERF_SERVER_PID $A_PID $B_PID

# wait for shutdown logs are written to files
sleep 1

echo "##########################################"
echo "A side logs:"
echo "##########################################"
cat a.out

echo "##########################################"
echo "B side logs:"
echo "##########################################"
cat b.out
rm a.out b.out
5 changes: 5 additions & 0 deletions e2e/forwarding/speedtest_udp.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash

cd "$(dirname "$0")"

./speedtest.sh --udp --bidir --bitrate=100MiB "$@"
29 changes: 28 additions & 1 deletion examples/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,11 @@ relay:

# Configure the private interface. Note: addr is baked into the nebula certificate
tun:
# When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root)
# When tun is disabled, a feature limited Nebula can be started without root privileges.
# In this limited mode, Nebula can
# - run a lighthouse node
# - offer access from and to the nebula network via port forwarding
# - respond to ping requests
disabled: false
# Name of the device. If not set, a default will be chosen by the OS.
# For macOS: if set, must be in the form `utun[0-9]+`.
Expand Down Expand Up @@ -368,3 +372,26 @@ firewall:
proto: tcp
group: remote_client
local_cidr: 192.168.100.1/24

# By using port port forwarding (port tunnels) its possible to establish connections
# from/into the nebula-network without using a tun/tap device and thus without requiring root access
# on the host. Port forwarding is only supported when setting "tun.disabled" is set to true.
# In this case, if a user-tun instead of a real one is instantiated to allow the port forwarding.
# IMPORTANT: For incoming tunnels, don't forget to also open the firewall for the relevant ports.
port_forwarding:
# Forces activation of the user tun, even when there is no rule specified.
# This can be useful, when rules are planned to be added later by reload.
# Reload config can't consider a change on tun-type.
enable_without_rules: false
outbound:
# format of listen- and dial-address: <host/ip>:<port>
#- listen_address: 127.0.0.1:3399
# dial_address: 192.168.100.92:4499
# format of protocols lists (yml-list): [tcp], [udp], [tcp, udp]
# protocols: [tcp, udp]
inbound:
# format of dial_address: <host/ip>:<port>
#- listen_port: 5599
# dial_address: 127.0.0.1:5599
# format of protocols lists (yml-list): [tcp], [udp], [tcp, udp]
# protocols: [tcp, udp]
6 changes: 5 additions & 1 deletion examples/go_service/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import (
"fmt"
"log"
"net"
"os"

"github.com/sirupsen/logrus"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/service"
)
Expand Down Expand Up @@ -59,7 +61,9 @@ pki:
if err := cfg.LoadString(configStr); err != nil {
return err
}
svc, err := service.New(&cfg)
l := logrus.New()
l.Out = os.Stdout
svc, err := service.New(&cfg, l)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
for {
n, err := reader.Read(packet)
if err != nil {
if errors.Is(err, os.ErrClosed) && f.closed.Load() {
if (errors.Is(err, os.ErrClosed) && f.closed.Load()) || errors.Is(err, io.EOF) {
return
}

Expand Down
Loading