Socket buffer controls to solve bufferbloat.

README.md

@@ -118,6 +118,13 @@ This field also has no effect if `AdminUID` isn't a valid UID or is empty.
 `KeepAlive` is the number of seconds to tell the OS to wait after no activity before sending TCP KeepAlive probes to the
 upstream proxy server. Zero or negative value disables it. Default is 0 (disabled).
 
+`LoopbackTcpSendBuffer` is the number of bytes to use for the tcp loopback send buffer. Use a low value like 4096 for a server-to-server bridge.
+
+`LoopbackTcpReceiveBuffer` is the number of bytes to use for the tcp loopback receive buffer. Use a low value like 4096 for a server-to-server bridge.
+
+These 2 options are not normally needed except when setting up a tcp server-to-server bridge using a shadowsocks or similar tcp server in the `ProxyBook` to reduce tcp performance degradation due to bufferbloat across the bridge.
+
+
 ### Client
 
 `UID` is your UID in base64.
@@ -179,6 +186,14 @@ more detectable as a proxy, but it will make the Cloak client detect internet in
 data, after which the connection will be closed by Cloak. Cloak will not enforce any timeout on TCP connections after it
 is established.
 
+`LoopbackTcpSendBuffer` is the number of bytes to use for the tcp loopback send buffer. Use a low value like 4096 to reduce upload bufferbloat on client.
+
+`LoopbackTcpReceiveBuffer` is the number of bytes to use for the tcp loopback receive buffer. Use a low value like 4096 to reduce download bufferbloat on client if the server is very close (low ping).
+
+`RemoteTcpSendBuffer` is the number of bytes to use for the tcp remote send buffer. Use a low value like 4096 for a server-to-server bridge.
+
+`RemoteTcpReceiveBuffer` is the number of bytes to use for the tcp remote receive buffer. Use a low value like 4096 for a server-to-server bridge.
+
 ## Setup
 
 ### Server

cmd/ck-client/ck-client.go

@@ -8,10 +8,10 @@ import (
 	"encoding/binary"
 	"flag"
 	"fmt"
+	"github.com/cbeuw/Cloak/internal/common"
 	"net"
 	"os"
-
+	"syscall"
-	"github.com/cbeuw/Cloak/internal/common"
 
 	"github.com/cbeuw/Cloak/internal/client"
 	mux "github.com/cbeuw/Cloak/internal/multiplex"
@@ -154,7 +154,37 @@ func main() {
 
 	var seshMaker func() *mux.Session
 
-	d := &net.Dialer{Control: protector, KeepAlive: remoteConfig.KeepAlive}
+	control := func(network string, address string, rawConn syscall.RawConn) error {
+		if !authInfo.Unordered {
+			sendBufferSize := remoteConfig.TcpSendBuffer
+			receiveBufferSize := remoteConfig.TcpReceiveBuffer
+
+			err := rawConn.Control(func(fd uintptr) {
+				if sendBufferSize > 0 {
+					log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize)
+					err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize)
+					if err != nil {
+						log.Errorf("setsocketopt SO_SNDBUF: %s\n", err)
+					}
+				}
+
+				if receiveBufferSize > 0 {
+					log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize)
+					err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize)
+					if err != nil {
+						log.Errorf("setsocketopt SO_RCVBUF: %s\n", err)
+					}
+				}
+			})
+			if err != nil {
+				panic(err)
+			}
+		}
+
+		return protector(network, address, rawConn)
+	}
+
+	d := &net.Dialer{Control: control, KeepAlive: remoteConfig.KeepAlive}
 
 	if adminUID != nil {
 		log.Infof("API base is %v", localConfig.LocalAddr)
@@ -199,8 +229,43 @@ func main() {
 	} else {
 		listener, err := net.Listen("tcp", localConfig.LocalAddr)
 		if err != nil {
-			log.Fatal(err)
+			panic(err)
 		}
+
+		tcpListener, ok := listener.(*net.TCPListener)
+		if !ok {
+			panic("Unknown listener type")
+		}
+
+		syscallConn, err := tcpListener.SyscallConn()
+		if err != nil {
+			panic(err)
+		}
+
+		sendBufferSize := localConfig.TcpSendBuffer
+		receiveBufferSize := localConfig.TcpReceiveBuffer
+
+		err = syscallConn.Control(func(fd uintptr) {
+			if sendBufferSize > 0 {
+				log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize)
+				err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize)
+				if err != nil {
+					log.Errorf("setsocketopt SO_SNDBUF: %s\n", err)
+				}
+			}
+
+			if receiveBufferSize > 0 {
+				log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize)
+				err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize)
+				if err != nil {
+					log.Errorf("setsocketopt SO_RCVBUF: %s\n", err)
+				}
+			}
+		})
+		if err != nil {
+			panic(err)
+		}
+
 		client.RouteTCP(listener, localConfig.Timeout, remoteConfig.Singleplex, seshMaker)
 	}
 }

internal/client/state.go

@@ -40,6 +40,10 @@ type RawConfig struct {
 	CDNWsUrlPath             string // nullable
 	StreamTimeout            int    // nullable
 	KeepAlive                int    // nullable
+	LoopbackTcpSendBuffer    int    // nullable
+	LoopbackTcpReceiveBuffer int    // nullable
+	RemoteTcpSendBuffer      int    // nullable
+	RemoteTcpReceiveBuffer   int    // nullable
 }
 
 type RemoteConnConfig struct {
@@ -48,12 +52,16 @@ type RemoteConnConfig struct {
 	KeepAlive        time.Duration
 	RemoteAddr       string
 	TransportMaker   func() Transport
+	TcpSendBuffer    int
+	TcpReceiveBuffer int
 }
 
 type LocalConnConfig struct {
 	LocalAddr        string
 	Timeout          time.Duration
 	MockDomainList   []string
+	TcpSendBuffer    int
+	TcpReceiveBuffer int
 }
 
 type AuthInfo struct {
@@ -83,7 +91,16 @@ func ssvToJson(ssv string) (ret []byte) {
 		r = strings.Replace(r, `\;`, `;`, -1)
 		return r
 	}
-	unquoted := []string{"NumConn", "StreamTimeout", "KeepAlive", "UDP"}
+	unquoted := []string{
+		"NumConn",
+		"StreamTimeout",
+		"KeepAlive",
+		"UDP",
+		"LoopbackTcpSendBuffer",
+		"LoopbackTcpReceiveBuffer",
+		"RemoteTcpSendBuffer",
+		"RemoteTcpReceiveBuffer",
+	}
 	lines := strings.Split(unescape(ssv), ";")
 	ret = []byte("{")
 	for _, ln := range lines {
@@ -277,5 +294,21 @@ func (raw *RawConfig) ProcessRawConfig(worldState common.WorldState) (local Loca
 		local.Timeout = time.Duration(raw.StreamTimeout) * time.Second
 	}
 
+	if raw.LoopbackTcpSendBuffer > 0 {
+		local.TcpSendBuffer = raw.LoopbackTcpSendBuffer
+	}
+
+	if raw.LoopbackTcpReceiveBuffer > 0 {
+		local.TcpReceiveBuffer = raw.LoopbackTcpReceiveBuffer
+	}
+
+	if raw.RemoteTcpSendBuffer > 0 {
+		remote.TcpSendBuffer = raw.RemoteTcpSendBuffer
+	}
+
+	if raw.RemoteTcpReceiveBuffer > 0 {
+		remote.TcpReceiveBuffer = raw.RemoteTcpReceiveBuffer
+	}
+
 	return
 }

internal/common/platformfd_darwin.go

@@ -0,0 +1,8 @@
+//go:build darwin
+// +build darwin
+
+package common
+
+func Platformfd(fd uintptr) int {
+	return int(fd)
+}

internal/common/platformfd_linux.go

@@ -0,0 +1,8 @@
+//go:build linux
+// +build linux
+
+package common
+
+func Platformfd(fd uintptr) int {
+	return int(fd)
+}

internal/common/platformfd_windows.go

@@ -0,0 +1,10 @@
+//go:build windows
+// +build windows
+
+package common
+
+import "syscall"
+
+func Platformfd(fd uintptr) syscall.Handle {
+	return syscall.Handle(fd)
+}

internal/server/state.go

@@ -9,10 +9,12 @@ import (
 	"net"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/cbeuw/Cloak/internal/common"
 	"github.com/cbeuw/Cloak/internal/server/usermanager"
+	log "github.com/sirupsen/logrus"
 )
 
 type RawConfig struct {
@@ -25,6 +27,8 @@ type RawConfig struct {
 	DatabasePath             string
 	KeepAlive                int
 	CncMode                  bool
+	LoopbackTcpSendBuffer    int
+	LoopbackTcpReceiveBuffer int
 }
 
 // State type stores the global state of the program
@@ -156,10 +160,46 @@ func InitState(preParse RawConfig, worldState common.WorldState) (sta *State, er
 		sta.Panel = MakeUserPanel(manager)
 	}
 
+	dialerControl := func(network, address string, c syscall.RawConn) error {
+		if !strings.HasPrefix(network, "tcp") {
+			return nil
+		}
+
+		ips, err := net.LookupHost(strings.Split(address, ":")[0])
+		if err != nil {
+			return err
+		}
+
+		for _, ipString := range ips {
+			ip := net.ParseIP(ipString)
+			if !ip.IsLoopback() {
+				return nil
+			}
+		}
+
+		return c.Control(func(fd uintptr) {
+			if preParse.LoopbackTcpSendBuffer > 0 {
+				log.Debugf("Setting loopback connection tcp send buffer: %d", preParse.LoopbackTcpSendBuffer)
+				err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, preParse.LoopbackTcpSendBuffer)
+				if err != nil {
+					log.Errorf("setsocketopt SO_SNDBUF: %s\n", err)
+				}
+			}
+
+			if preParse.LoopbackTcpReceiveBuffer > 0 {
+				log.Debugf("Setting loopback connection tcp receive buffer: %d", preParse.LoopbackTcpReceiveBuffer)
+				err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, preParse.LoopbackTcpReceiveBuffer)
+				if err != nil {
+					log.Errorf("setsocketopt SO_RCVBUF: %s\n", err)
+				}
+			}
+		})
+	}
+
 	if preParse.KeepAlive <= 0 {
-		sta.ProxyDialer = &net.Dialer{KeepAlive: -1}
+		sta.ProxyDialer = &net.Dialer{KeepAlive: -1, Control: dialerControl}
 	} else {
-		sta.ProxyDialer = &net.Dialer{KeepAlive: time.Duration(preParse.KeepAlive) * time.Second}
+		sta.ProxyDialer = &net.Dialer{KeepAlive: time.Duration(preParse.KeepAlive) * time.Second, Control: dialerControl}
 	}
 
 	sta.RedirHost, sta.RedirPort, err = parseRedirAddr(preParse.RedirAddr)