Socket buffer controls to solve bufferbloat.

README.md

@@ -118,6 +118,13 @@ This field also has no effect if `AdminUID` isn't a valid UID or is empty.
 `KeepAlive` is the number of seconds to tell the OS to wait after no activity before sending TCP KeepAlive probes to the
 upstream proxy server. Zero or negative value disables it. Default is 0 (disabled).
 
+`LoopbackTcpSendBuffer` is the number of bytes to use for the tcp loopback send buffer. Use a low value like 4096 for a server-to-server bridge.
+
+`LoopbackTcpReceiveBuffer` is the number of bytes to use for the tcp loopback receive buffer. Use a low value like 4096 for a server-to-server bridge.
+
+These 2 options are not normally needed except when setting up a tcp server-to-server bridge using a shadowsocks or similar tcp server in the `ProxyBook` to reduce tcp performance degradation due to bufferbloat across the bridge.
+
+
 ### Client
 
 `UID` is your UID in base64.
@@ -179,6 +186,14 @@ more detectable as a proxy, but it will make the Cloak client detect internet in
 data, after which the connection will be closed by Cloak. Cloak will not enforce any timeout on TCP connections after it
 is established.
 
+`LoopbackTcpSendBuffer` is the number of bytes to use for the tcp loopback send buffer. Use a low value like 4096 to reduce upload bufferbloat on client.
+
+`LoopbackTcpReceiveBuffer` is the number of bytes to use for the tcp loopback receive buffer. Use a low value like 4096 to reduce download bufferbloat on client if the server is very close (low ping).
+
+`RemoteTcpSendBuffer` is the number of bytes to use for the tcp remote send buffer. Use a low value like 4096 for a server-to-server bridge.
+
+`RemoteTcpReceiveBuffer` is the number of bytes to use for the tcp remote receive buffer. Use a low value like 4096 for a server-to-server bridge.
+
 ## Setup
 
 ### Server

cmd/ck-client/ck-client.go

@@ -8,10 +8,10 @@ import (
 	"encoding/binary"
 	"flag"
 	"fmt"
+	"github.com/cbeuw/Cloak/internal/common"
 	"net"
 	"os"
-
+	"syscall"
-	"github.com/cbeuw/Cloak/internal/common"
 
 	"github.com/cbeuw/Cloak/internal/client"
 	mux "github.com/cbeuw/Cloak/internal/multiplex"
@@ -154,7 +154,37 @@ func main() {
 
 	var seshMaker func() *mux.Session
 
-	d := &net.Dialer{Control: protector, KeepAlive: remoteConfig.KeepAlive}
+	control := func(network string, address string, rawConn syscall.RawConn) error {
+		if !authInfo.Unordered {
+			sendBufferSize := remoteConfig.TcpSendBuffer
+			receiveBufferSize := remoteConfig.TcpReceiveBuffer
+
+			err := rawConn.Control(func(fd uintptr) {
+				if sendBufferSize > 0 {
+					log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize)
+					err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize)
+					if err != nil {
+						log.Errorf("setsocketopt SO_SNDBUF: %s\n", err)
+					}
+				}
+
+				if receiveBufferSize > 0 {
+					log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize)
+					err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize)
+					if err != nil {
+						log.Errorf("setsocketopt SO_RCVBUF: %s\n", err)
+					}
+				}
+			})
+			if err != nil {
+				panic(err)
+			}
+		}
+
+		return protector(network, address, rawConn)
+	}
+
+	d := &net.Dialer{Control: control, KeepAlive: remoteConfig.KeepAlive}
 
 	if adminUID != nil {
 		log.Infof("API base is %v", localConfig.LocalAddr)
@@ -199,8 +229,43 @@ func main() {
 	} else {
 		listener, err := net.Listen("tcp", localConfig.LocalAddr)
 		if err != nil {
-			log.Fatal(err)
+			panic(err)
 		}
+
+		tcpListener, ok := listener.(*net.TCPListener)
+		if !ok {
+			panic("Unknown listener type")
+		}
+
+		syscallConn, err := tcpListener.SyscallConn()
+		if err != nil {
+			panic(err)
+		}
+
+		sendBufferSize := localConfig.TcpSendBuffer
+		receiveBufferSize := localConfig.TcpReceiveBuffer
+
+		err = syscallConn.Control(func(fd uintptr) {
+			if sendBufferSize > 0 {
+				log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize)
+				err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize)
+				if err != nil {
+					log.Errorf("setsocketopt SO_SNDBUF: %s\n", err)
+				}
+			}
+
+			if receiveBufferSize > 0 {
+				log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize)
+				err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize)
+				if err != nil {
+					log.Errorf("setsocketopt SO_RCVBUF: %s\n", err)
+				}
+			}
+		})
+		if err != nil {
+			panic(err)
+		}
+
 		client.RouteTCP(listener, localConfig.Timeout, remoteConfig.Singleplex, seshMaker)
 	}
 }

internal/client/state.go

@@ -33,27 +33,35 @@ type RawConfig struct {
 	RemotePort       string   // jsonOptional
 	AlternativeNames []string // jsonOptional
 	// defaults set in ProcessRawConfig
-	UDP           bool   // nullable
+	UDP                      bool   // nullable
-	BrowserSig    string // nullable
+	BrowserSig               string // nullable
-	Transport     string // nullable
+	Transport                string // nullable
-	CDNOriginHost string // nullable
+	CDNOriginHost            string // nullable
-	CDNWsUrlPath  string // nullable
+	CDNWsUrlPath             string // nullable
-	StreamTimeout int    // nullable
+	StreamTimeout            int    // nullable
-	KeepAlive     int    // nullable
+	KeepAlive                int    // nullable
+	LoopbackTcpSendBuffer    int    // nullable
+	LoopbackTcpReceiveBuffer int    // nullable
+	RemoteTcpSendBuffer      int    // nullable
+	RemoteTcpReceiveBuffer   int    // nullable
 }
 
 type RemoteConnConfig struct {
-	Singleplex     bool
+	Singleplex       bool
-	NumConn        int
+	NumConn          int
-	KeepAlive      time.Duration
+	KeepAlive        time.Duration
-	RemoteAddr     string
+	RemoteAddr       string
-	TransportMaker func() Transport
+	TransportMaker   func() Transport
+	TcpSendBuffer    int
+	TcpReceiveBuffer int
 }
 
 type LocalConnConfig struct {
-	LocalAddr      string
+	LocalAddr        string
-	Timeout        time.Duration
+	Timeout          time.Duration
-	MockDomainList []string
+	MockDomainList   []string
+	TcpSendBuffer    int
+	TcpReceiveBuffer int
 }
 
 type AuthInfo struct {
@@ -83,7 +91,16 @@ func ssvToJson(ssv string) (ret []byte) {
 		r = strings.Replace(r, `\;`, `;`, -1)
 		return r
 	}
-	unquoted := []string{"NumConn", "StreamTimeout", "KeepAlive", "UDP"}
+	unquoted := []string{
+		"NumConn",
+		"StreamTimeout",
+		"KeepAlive",
+		"UDP",
+		"LoopbackTcpSendBuffer",
+		"LoopbackTcpReceiveBuffer",
+		"RemoteTcpSendBuffer",
+		"RemoteTcpReceiveBuffer",
+	}
 	lines := strings.Split(unescape(ssv), ";")
 	ret = []byte("{")
 	for _, ln := range lines {
@@ -277,5 +294,21 @@ func (raw *RawConfig) ProcessRawConfig(worldState common.WorldState) (local Loca
 		local.Timeout = time.Duration(raw.StreamTimeout) * time.Second
 	}
 
+	if raw.LoopbackTcpSendBuffer > 0 {
+		local.TcpSendBuffer = raw.LoopbackTcpSendBuffer
+	}
+
+	if raw.LoopbackTcpReceiveBuffer > 0 {
+		local.TcpReceiveBuffer = raw.LoopbackTcpReceiveBuffer
+	}
+
+	if raw.RemoteTcpSendBuffer > 0 {
+		remote.TcpSendBuffer = raw.RemoteTcpSendBuffer
+	}
+
+	if raw.RemoteTcpReceiveBuffer > 0 {
+		remote.TcpReceiveBuffer = raw.RemoteTcpReceiveBuffer
+	}
+
 	return
 }

internal/common/platformfd_darwin.go

@@ -0,0 +1,8 @@
+//go:build darwin
+// +build darwin
+
+package common
+
+func Platformfd(fd uintptr) int {
+	return int(fd)
+}

internal/common/platformfd_linux.go

@@ -0,0 +1,8 @@
+//go:build linux
+// +build linux
+
+package common
+
+func Platformfd(fd uintptr) int {
+	return int(fd)
+}

internal/common/platformfd_windows.go

@@ -0,0 +1,10 @@
+//go:build windows
+// +build windows
+
+package common
+
+import "syscall"
+
+func Platformfd(fd uintptr) syscall.Handle {
+	return syscall.Handle(fd)
+}

internal/server/state.go

@@ -9,22 +9,26 @@ import (
 	"net"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/cbeuw/Cloak/internal/common"
 	"github.com/cbeuw/Cloak/internal/server/usermanager"
+	log "github.com/sirupsen/logrus"
 )
 
 type RawConfig struct {
-	ProxyBook    map[string][]string
+	ProxyBook                map[string][]string
-	BindAddr     []string
+	BindAddr                 []string
-	BypassUID    [][]byte
+	BypassUID                [][]byte
-	RedirAddr    string
+	RedirAddr                string
-	PrivateKey   []byte
+	PrivateKey               []byte
-	AdminUID     []byte
+	AdminUID                 []byte
-	DatabasePath string
+	DatabasePath             string
-	KeepAlive    int
+	KeepAlive                int
-	CncMode      bool
+	CncMode                  bool
+	LoopbackTcpSendBuffer    int
+	LoopbackTcpReceiveBuffer int
 }
 
 // State type stores the global state of the program
@@ -156,10 +160,46 @@ func InitState(preParse RawConfig, worldState common.WorldState) (sta *State, er
 		sta.Panel = MakeUserPanel(manager)
 	}
 
+	dialerControl := func(network, address string, c syscall.RawConn) error {
+		if !strings.HasPrefix(network, "tcp") {
+			return nil
+		}
+
+		ips, err := net.LookupHost(strings.Split(address, ":")[0])
+		if err != nil {
+			return err
+		}
+
+		for _, ipString := range ips {
+			ip := net.ParseIP(ipString)
+			if !ip.IsLoopback() {
+				return nil
+			}
+		}
+
+		return c.Control(func(fd uintptr) {
+			if preParse.LoopbackTcpSendBuffer > 0 {
+				log.Debugf("Setting loopback connection tcp send buffer: %d", preParse.LoopbackTcpSendBuffer)
+				err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, preParse.LoopbackTcpSendBuffer)
+				if err != nil {
+					log.Errorf("setsocketopt SO_SNDBUF: %s\n", err)
+				}
+			}
+
+			if preParse.LoopbackTcpReceiveBuffer > 0 {
+				log.Debugf("Setting loopback connection tcp receive buffer: %d", preParse.LoopbackTcpReceiveBuffer)
+				err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, preParse.LoopbackTcpReceiveBuffer)
+				if err != nil {
+					log.Errorf("setsocketopt SO_RCVBUF: %s\n", err)
+				}
+			}
+		})
+	}
+
 	if preParse.KeepAlive <= 0 {
-		sta.ProxyDialer = &net.Dialer{KeepAlive: -1}
+		sta.ProxyDialer = &net.Dialer{KeepAlive: -1, Control: dialerControl}
 	} else {
-		sta.ProxyDialer = &net.Dialer{KeepAlive: time.Duration(preParse.KeepAlive) * time.Second}
+		sta.ProxyDialer = &net.Dialer{KeepAlive: time.Duration(preParse.KeepAlive) * time.Second, Control: dialerControl}
 	}
 
 	sta.RedirHost, sta.RedirPort, err = parseRedirAddr(preParse.RedirAddr)