From 18355d06ceb6c232c499003b5b57c521c58dea34 Mon Sep 17 00:00:00 2001 From: notsure2 Date: Tue, 12 Dec 2023 01:45:48 +0200 Subject: [PATCH 1/2] Socket buffer controls to solve bufferbloat. --- README.md | 15 ++++++ cmd/ck-client/ck-client.go | 73 +++++++++++++++++++++++++-- internal/client/state.go | 65 ++++++++++++++++++------ internal/common/platformfd_darwin.go | 8 +++ internal/common/platformfd_linux.go | 8 +++ internal/common/platformfd_windows.go | 10 ++++ internal/server/state.go | 62 +++++++++++++++++++---- 7 files changed, 210 insertions(+), 31 deletions(-) create mode 100644 internal/common/platformfd_darwin.go create mode 100644 internal/common/platformfd_linux.go create mode 100644 internal/common/platformfd_windows.go diff --git a/README.md b/README.md index e4c0511..8cd63f3 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,13 @@ This field also has no effect if `AdminUID` isn't a valid UID or is empty. `KeepAlive` is the number of seconds to tell the OS to wait after no activity before sending TCP KeepAlive probes to the upstream proxy server. Zero or negative value disables it. Default is 0 (disabled). +`LoopbackTcpSendBuffer` is the number of bytes to use for the tcp loopback send buffer. Use a low value like 4096 for a server-to-server bridge. + +`LoopbackTcpReceiveBuffer` is the number of bytes to use for the tcp loopback receive buffer. Use a low value like 4096 for a server-to-server bridge. + +These 2 options are not normally needed except when setting up a tcp server-to-server bridge using a shadowsocks or similar tcp server in the `ProxyBook` to reduce tcp performance degradation due to bufferbloat across the bridge. + + ### Client `UID` is your UID in base64. @@ -179,6 +186,14 @@ more detectable as a proxy, but it will make the Cloak client detect internet in data, after which the connection will be closed by Cloak. Cloak will not enforce any timeout on TCP connections after it is established. +`LoopbackTcpSendBuffer` is the number of bytes to use for the tcp loopback send buffer. Use a low value like 4096 to reduce upload bufferbloat on client. + +`LoopbackTcpReceiveBuffer` is the number of bytes to use for the tcp loopback receive buffer. Use a low value like 4096 to reduce download bufferbloat on client if the server is very close (low ping). + +`RemoteTcpSendBuffer` is the number of bytes to use for the tcp remote send buffer. Use a low value like 4096 for a server-to-server bridge. + +`RemoteTcpReceiveBuffer` is the number of bytes to use for the tcp remote receive buffer. Use a low value like 4096 for a server-to-server bridge. + ## Setup ### Server diff --git a/cmd/ck-client/ck-client.go b/cmd/ck-client/ck-client.go index 1a43651..f28213d 100644 --- a/cmd/ck-client/ck-client.go +++ b/cmd/ck-client/ck-client.go @@ -8,10 +8,10 @@ import ( "encoding/binary" "flag" "fmt" + "github.com/cbeuw/Cloak/internal/common" "net" "os" - - "github.com/cbeuw/Cloak/internal/common" + "syscall" "github.com/cbeuw/Cloak/internal/client" mux "github.com/cbeuw/Cloak/internal/multiplex" @@ -154,7 +154,37 @@ func main() { var seshMaker func() *mux.Session - d := &net.Dialer{Control: protector, KeepAlive: remoteConfig.KeepAlive} + control := func(network string, address string, rawConn syscall.RawConn) error { + if !authInfo.Unordered { + sendBufferSize := remoteConfig.TcpSendBuffer + receiveBufferSize := remoteConfig.TcpReceiveBuffer + + err := rawConn.Control(func(fd uintptr) { + if sendBufferSize > 0 { + log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize) + err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize) + if err != nil { + log.Errorf("setsocketopt SO_SNDBUF: %s\n", err) + } + } + + if receiveBufferSize > 0 { + log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize) + err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize) + if err != nil { + log.Errorf("setsocketopt SO_RCVBUF: %s\n", err) + } + } + }) + if err != nil { + panic(err) + } + } + + return protector(network, address, rawConn) + } + + d := &net.Dialer{Control: control, KeepAlive: remoteConfig.KeepAlive} if adminUID != nil { log.Infof("API base is %v", localConfig.LocalAddr) @@ -199,8 +229,43 @@ func main() { } else { listener, err := net.Listen("tcp", localConfig.LocalAddr) if err != nil { - log.Fatal(err) + panic(err) } + + tcpListener, ok := listener.(*net.TCPListener) + if !ok { + panic("Unknown listener type") + } + + syscallConn, err := tcpListener.SyscallConn() + if err != nil { + panic(err) + } + + sendBufferSize := localConfig.TcpSendBuffer + receiveBufferSize := localConfig.TcpReceiveBuffer + + err = syscallConn.Control(func(fd uintptr) { + if sendBufferSize > 0 { + log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize) + err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize) + if err != nil { + log.Errorf("setsocketopt SO_SNDBUF: %s\n", err) + } + } + + if receiveBufferSize > 0 { + log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize) + err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize) + if err != nil { + log.Errorf("setsocketopt SO_RCVBUF: %s\n", err) + } + } + }) + if err != nil { + panic(err) + } + client.RouteTCP(listener, localConfig.Timeout, remoteConfig.Singleplex, seshMaker) } } diff --git a/internal/client/state.go b/internal/client/state.go index c3d1ded..46f72ce 100644 --- a/internal/client/state.go +++ b/internal/client/state.go @@ -33,27 +33,35 @@ type RawConfig struct { RemotePort string // jsonOptional AlternativeNames []string // jsonOptional // defaults set in ProcessRawConfig - UDP bool // nullable - BrowserSig string // nullable - Transport string // nullable - CDNOriginHost string // nullable - CDNWsUrlPath string // nullable - StreamTimeout int // nullable - KeepAlive int // nullable + UDP bool // nullable + BrowserSig string // nullable + Transport string // nullable + CDNOriginHost string // nullable + CDNWsUrlPath string // nullable + StreamTimeout int // nullable + KeepAlive int // nullable + LoopbackTcpSendBuffer int // nullable + LoopbackTcpReceiveBuffer int // nullable + RemoteTcpSendBuffer int // nullable + RemoteTcpReceiveBuffer int // nullable } type RemoteConnConfig struct { - Singleplex bool - NumConn int - KeepAlive time.Duration - RemoteAddr string - TransportMaker func() Transport + Singleplex bool + NumConn int + KeepAlive time.Duration + RemoteAddr string + TransportMaker func() Transport + TcpSendBuffer int + TcpReceiveBuffer int } type LocalConnConfig struct { - LocalAddr string - Timeout time.Duration - MockDomainList []string + LocalAddr string + Timeout time.Duration + MockDomainList []string + TcpSendBuffer int + TcpReceiveBuffer int } type AuthInfo struct { @@ -83,7 +91,16 @@ func ssvToJson(ssv string) (ret []byte) { r = strings.Replace(r, `\;`, `;`, -1) return r } - unquoted := []string{"NumConn", "StreamTimeout", "KeepAlive", "UDP"} + unquoted := []string{ + "NumConn", + "StreamTimeout", + "KeepAlive", + "UDP", + "LoopbackTcpSendBuffer", + "LoopbackTcpReceiveBuffer", + "RemoteTcpSendBuffer", + "RemoteTcpReceiveBuffer", + } lines := strings.Split(unescape(ssv), ";") ret = []byte("{") for _, ln := range lines { @@ -277,5 +294,21 @@ func (raw *RawConfig) ProcessRawConfig(worldState common.WorldState) (local Loca local.Timeout = time.Duration(raw.StreamTimeout) * time.Second } + if raw.LoopbackTcpSendBuffer > 0 { + local.TcpSendBuffer = raw.LoopbackTcpSendBuffer + } + + if raw.LoopbackTcpReceiveBuffer > 0 { + local.TcpReceiveBuffer = raw.LoopbackTcpReceiveBuffer + } + + if raw.RemoteTcpSendBuffer > 0 { + remote.TcpSendBuffer = raw.RemoteTcpSendBuffer + } + + if raw.RemoteTcpReceiveBuffer > 0 { + remote.TcpReceiveBuffer = raw.RemoteTcpReceiveBuffer + } + return } diff --git a/internal/common/platformfd_darwin.go b/internal/common/platformfd_darwin.go new file mode 100644 index 0000000..cfdb699 --- /dev/null +++ b/internal/common/platformfd_darwin.go @@ -0,0 +1,8 @@ +//go:build darwin +// +build darwin + +package common + +func Platformfd(fd uintptr) int { + return int(fd) +} diff --git a/internal/common/platformfd_linux.go b/internal/common/platformfd_linux.go new file mode 100644 index 0000000..bc56635 --- /dev/null +++ b/internal/common/platformfd_linux.go @@ -0,0 +1,8 @@ +//go:build linux +// +build linux + +package common + +func Platformfd(fd uintptr) int { + return int(fd) +} diff --git a/internal/common/platformfd_windows.go b/internal/common/platformfd_windows.go new file mode 100644 index 0000000..bb0948b --- /dev/null +++ b/internal/common/platformfd_windows.go @@ -0,0 +1,10 @@ +//go:build windows +// +build windows + +package common + +import "syscall" + +func Platformfd(fd uintptr) syscall.Handle { + return syscall.Handle(fd) +} diff --git a/internal/server/state.go b/internal/server/state.go index 457dc96..62a1121 100644 --- a/internal/server/state.go +++ b/internal/server/state.go @@ -9,22 +9,26 @@ import ( "net" "strings" "sync" + "syscall" "time" "github.com/cbeuw/Cloak/internal/common" "github.com/cbeuw/Cloak/internal/server/usermanager" + log "github.com/sirupsen/logrus" ) type RawConfig struct { - ProxyBook map[string][]string - BindAddr []string - BypassUID [][]byte - RedirAddr string - PrivateKey []byte - AdminUID []byte - DatabasePath string - KeepAlive int - CncMode bool + ProxyBook map[string][]string + BindAddr []string + BypassUID [][]byte + RedirAddr string + PrivateKey []byte + AdminUID []byte + DatabasePath string + KeepAlive int + CncMode bool + LoopbackTcpSendBuffer int + LoopbackTcpReceiveBuffer int } // State type stores the global state of the program @@ -156,10 +160,46 @@ func InitState(preParse RawConfig, worldState common.WorldState) (sta *State, er sta.Panel = MakeUserPanel(manager) } + dialerControl := func(network, address string, c syscall.RawConn) error { + if !strings.HasPrefix(network, "tcp") { + return nil + } + + ips, err := net.LookupHost(strings.Split(address, ":")[0]) + if err != nil { + return err + } + + for _, ipString := range ips { + ip := net.ParseIP(ipString) + if !ip.IsLoopback() { + return nil + } + } + + return c.Control(func(fd uintptr) { + if preParse.LoopbackTcpSendBuffer > 0 { + log.Debugf("Setting loopback connection tcp send buffer: %d", preParse.LoopbackTcpSendBuffer) + err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, preParse.LoopbackTcpSendBuffer) + if err != nil { + log.Errorf("setsocketopt SO_SNDBUF: %s\n", err) + } + } + + if preParse.LoopbackTcpReceiveBuffer > 0 { + log.Debugf("Setting loopback connection tcp receive buffer: %d", preParse.LoopbackTcpReceiveBuffer) + err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, preParse.LoopbackTcpReceiveBuffer) + if err != nil { + log.Errorf("setsocketopt SO_RCVBUF: %s\n", err) + } + } + }) + } + if preParse.KeepAlive <= 0 { - sta.ProxyDialer = &net.Dialer{KeepAlive: -1} + sta.ProxyDialer = &net.Dialer{KeepAlive: -1, Control: dialerControl} } else { - sta.ProxyDialer = &net.Dialer{KeepAlive: time.Duration(preParse.KeepAlive) * time.Second} + sta.ProxyDialer = &net.Dialer{KeepAlive: time.Duration(preParse.KeepAlive) * time.Second, Control: dialerControl} } sta.RedirHost, sta.RedirPort, err = parseRedirAddr(preParse.RedirAddr) From c3cde8d8af14e211487030f1797d0592488223de Mon Sep 17 00:00:00 2001 From: notsure2 Date: Thu, 21 Dec 2023 23:28:57 +0200 Subject: [PATCH 2/2] Fix: buffer should be set even in unordered mode because in the end it's tcp --- cmd/ck-client/ck-client.go | 40 ++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/cmd/ck-client/ck-client.go b/cmd/ck-client/ck-client.go index f28213d..90b67e2 100644 --- a/cmd/ck-client/ck-client.go +++ b/cmd/ck-client/ck-client.go @@ -155,30 +155,28 @@ func main() { var seshMaker func() *mux.Session control := func(network string, address string, rawConn syscall.RawConn) error { - if !authInfo.Unordered { - sendBufferSize := remoteConfig.TcpSendBuffer - receiveBufferSize := remoteConfig.TcpReceiveBuffer + sendBufferSize := remoteConfig.TcpSendBuffer + receiveBufferSize := remoteConfig.TcpReceiveBuffer - err := rawConn.Control(func(fd uintptr) { - if sendBufferSize > 0 { - log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize) - err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize) - if err != nil { - log.Errorf("setsocketopt SO_SNDBUF: %s\n", err) - } + err := rawConn.Control(func(fd uintptr) { + if sendBufferSize > 0 { + log.Debugf("Setting remote connection tcp send buffer: %d", sendBufferSize) + err := syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize) + if err != nil { + log.Errorf("setsocketopt SO_SNDBUF: %s\n", err) } - - if receiveBufferSize > 0 { - log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize) - err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize) - if err != nil { - log.Errorf("setsocketopt SO_RCVBUF: %s\n", err) - } - } - }) - if err != nil { - panic(err) } + + if receiveBufferSize > 0 { + log.Debugf("Setting remote connection tcp receive buffer: %d", receiveBufferSize) + err = syscall.SetsockoptInt(common.Platformfd(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, receiveBufferSize) + if err != nil { + log.Errorf("setsocketopt SO_RCVBUF: %s\n", err) + } + } + }) + if err != nil { + panic(err) } return protector(network, address, rawConn)