diff --git a/README.md b/README.md index db52aa3..30b7d91 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,25 @@ # chisel + Chisel is an HTTP client and server which acts as a TCP proxy. Chisel useful in situations where you only have access to HTTP, for example – behind a corporate firewall. Chisel is very similar to [crowbar](https://github.com/q3k/crowbar) though achieves **much** higher [performance](#performance). **Warning** This is beta software. +![overview](https://docs.google.com/drawings/d/1p53VWxzGNfy8rjr-mW8pvisJmhkoLl82vAgctO_6f1w/pub?w=960&h=720) + ### Install -Server +**Binaries** + +See [Releases](releases/) + +**Source** ``` +# Server $ go get -v github.com/jpillora/chisel/chiseld -``` - -Forwarder - -``` +# Client $ go get -v github.com/jpillora/chisel/chisel-forward ``` @@ -54,23 +59,27 @@ default proxy is pointing at http://example.com. ``` $ chiseld --help - Usage: chiseld [--host 0.0.0.0] [--port 8080] [--auth AUTH] [--proxy PROXY] + Usage: chiseld [options] - host defines the HTTP listening host – the - network interface (defaults to 0.0.0.0). You - may also set the HOST environment variable. + Options: - port defines the HTTP listening port (defaults - to 8080). This option falls back to the PORT - environment variable. + --host, Defines the HTTP listening host – the network interface + (defaults to 0.0.0.0). You may also set the HOST environment + variable. - auth specifies the authentication string - the client must provide to attain access. This - option falls back to the AUTH environment variable. + --port, Defines the HTTP listening port (defaults to 8080). You + may also set the PORT environment variable. - proxy specifies the default proxy target to use - when chiseld receives a normal HTTP request. This - option falls back to the PROXY environment variable. + --auth, Specifies the exact authentication string the client must + provide to attain access. You may also set the AUTH environment + variable. + + --proxy, Specifies the default proxy target to use when chiseld + receives a normal HTTP request. + + -v, Enable verbose logging + + --version, Display version Read more: https://github.com/jpillora/chisel @@ -79,18 +88,16 @@ $ chiseld --help ``` $ chisel-forward --help - Usage: chisel-forward [--auth AUTH] server remote [remote] [remote] ... + Usage: chisel-forward [options] server remote [remote] [remote] ... - auth specifies the optional authentication string - used by the server. + server is the URL to the chiseld server. - server is the URL of the chiseld server. + remotes are remote connections tunneled through the server, each of + which come in the form: - remote is a remote connection via the server, which - comes in the form: ::: - * Only remote-port is required. + * remote-port is required. * local-port defaults to remote-port. * local-host defaults to 0.0.0.0 (all interfaces). * remote-host defaults to 0.0.0.0 (server localhost). @@ -102,11 +109,20 @@ $ chisel-forward --help 3000:google.com:80 192.168.0.5:3000:google.com:80 + Options: + + --auth AUTH, Specifies the optional authentication string used by + the server. + + -v, Enable verbose logging + + --version, Display version + Read more: https://github.com/jpillora/chisel ``` -Eventually, a programmatic API will be documented and available, if you're keen see the `main.go` files in each sub-package. +See also: [programmatic API](https://github.com/jpillora/chisel/wiki/Programmatic-Usage). ### Security @@ -116,71 +132,63 @@ Currently, you can secure your traffic by using HTTPS, which can only be done by With [crowbar](https://github.com/q3k/crowbar), a connection is tunnelled by repeatedly querying the server with updates. This results in a large amount of HTTP and TCP connection overhead. Chisel overcomes this using WebSockets combined with [Yamux](https://github.com/hashicorp/yamux) to create hundreds of SDPY/HTTP2 like logical connections, resulting in **one** TCP connection per client. -In this unscientific test, we have: +In this simple benchmark, we have: ``` -curl -> http tunnel client -> http tunnel server -> file server + (direct) + .--------------->----------------. + / chisel chisel \ +request--->client:2001--->server:2002---->fileserver:3000 + \ / + '--> crowbar:4001--->crowbar:4002' + client server ``` -*Tab 1 (local file server)* +Note, we're using an in-memory "file" server on localhost for these tests + +*direct* ``` -$ npm i -g serve -$ serve -p 4000 +:3000 => 1 bytes in 1.008883ms +:3000 => 10 bytes in 543.198µs +:3000 => 100 bytes in 675.957µs +:3000 => 1000 bytes in 584.13µs +:3000 => 10000 bytes in 580.56µs +:3000 => 100000 bytes in 743.902µs +:3000 => 1000000 bytes in 1.962673ms +:3000 => 10000000 bytes in 19.192986ms +:3000 => 100000000 bytes in 158.428239ms ``` -*Tab 2 (tunnel server)* +`chisel` ``` -$ echo -ne "foo:bar" > userfile -$ crowbard -listen="0.0.0.0:8080" -userfile=./userfile +:2001 => 1 bytes in 1.334661ms +:2001 => 10 bytes in 807.797µs +:2001 => 100 bytes in 763.728µs +:2001 => 1000 bytes in 1.029811ms +:2001 => 10000 bytes in 840.247µs +:2001 => 100000 bytes in 1.647748ms +:2001 => 1000000 bytes in 3.495904ms +:2001 => 10000000 bytes in 22.298904ms +:2001 => 100000000 bytes in 255.410448ms ``` -*Tab 3 (tunnel client)* +`crowbar` ``` -$ crowbar-forward -local=0.0.0.0:3000 -server http://localhost:8080 -remote localhost:4000 -username foo -password bar +:4001 => 1 bytes in 3.335797ms +:4001 => 10 bytes in 1.453007ms +:4001 => 100 bytes in 1.811727ms +:4001 => 1000 bytes in 1.621525ms +:4001 => 10000 bytes in 5.20729ms +:4001 => 100000 bytes in 38.461926ms +:4001 => 1000000 bytes in 358.784864ms +:4001 => 10000000 bytes in 3.603206487s +:4001 => 100000000 bytes in 36.332395213s ``` -*Tab 4 (transfer test)* - -``` -$ time curl -s "127.0.0.1:3000/largefile.bin" > /dev/null - 74.74 real 2.37 user 6.74 sys -``` - -Here, we see `largefile.bin` (~200MB) is transferred in **1m14s** (along with high CPU utilisation). - -Enter `chisel`, lets swap in `chiseld` and `chisel-forward` - -*Tab 2 (tunnel server)* - -``` -$ chiseld --auth foo -``` - -*Tab 3 (tunnel client)* - -``` -$ chisel-forward --auth foo localhost:8080 3000:4000 -2015/02/27 16:13:43 Connected to http://localhost:8080 -2015/02/27 16:13:43 Proxy 0.0.0.0:3000 => 0.0.0.0:4000 enabled -``` - -And now we'll run the test again - -``` -$ time curl -s "127.0.0.1:3000/largefile.bin" > /dev/null - 0.60 real 0.05 user 0.14 sys -``` - -Here, the same file was transferred in **0.6s** - -*Note: Real benchmarks are on the Todo* - -### Overview - -![overview](https://docs.google.com/drawings/d/1p53VWxzGNfy8rjr-mW8pvisJmhkoLl82vAgctO_6f1w/pub?w=960&h=720) +See [test/](tree/master/test/) ### Known Issues @@ -213,7 +221,7 @@ Here, the same file was transferred in **0.6s** #### MIT License -Copyright © 2014 Jaime Pillora <dev@jpillora.com> +Copyright © 2015 Jaime Pillora <dev@jpillora.com> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/chisel-forward/client/client.go b/chisel-forward/client/client.go index 7710a25..ee31056 100644 --- a/chisel-forward/client/client.go +++ b/chisel-forward/client/client.go @@ -1,4 +1,4 @@ -package client +package chiselclient import ( "errors" @@ -26,7 +26,7 @@ type Client struct { runningc chan error } -func NewClient(auth, server string, remotes []string) (*Client, error) { +func NewClient(auth, server string, remotes ...string) (*Client, error) { //apply default scheme if !strings.HasPrefix(server, "http") { @@ -95,7 +95,7 @@ func (c *Client) start() { //proxies all use this function openStream := func() (net.Conn, error) { if c.session == nil || c.session.IsClosed() { - return nil, c.Errorf("no c.session") + return nil, c.Errorf("no session available") } stream, err := c.session.Open() if err != nil { @@ -184,5 +184,8 @@ func (c *Client) Wait() error { //Close manual stops the client func (c *Client) Close() error { c.running = false + if c.session == nil { + return nil + } return c.session.Close() } diff --git a/chisel-forward/client/proxy.go b/chisel-forward/client/proxy.go index e0397e1..01c5047 100644 --- a/chisel-forward/client/proxy.go +++ b/chisel-forward/client/proxy.go @@ -1,4 +1,4 @@ -package client +package chiselclient import ( "encoding/binary" diff --git a/chisel-forward/main.go b/chisel-forward/main.go index 3d41cb3..866e916 100644 --- a/chisel-forward/main.go +++ b/chisel-forward/main.go @@ -9,17 +9,19 @@ import ( "github.com/jpillora/chisel/chisel-forward/client" ) -const help = ` +var VERSION string = "0.0.0-src" //set via ldflags +var help = ` Usage: chisel-forward [options] server remote [remote] [remote] ... server is the URL to the chiseld server. - remotes are remote connections tunneled through the server, each - of which come in the form: + remotes are remote connections tunneled through the server, each of + which come in the form: + ::: - * Only remote-port is required. + * remote-port is required. * local-port defaults to remote-port. * local-host defaults to 0.0.0.0 (all interfaces). * remote-host defaults to 0.0.0.0 (server localhost). @@ -33,23 +35,31 @@ const help = ` Options: - --auth AUTH - auth specifies the optional authentication string - used by the server. + --auth AUTH, Specifies the optional authentication string used by + the server. - -v enable verbose logging + -v, Enable verbose logging + + --version, Display version (` + VERSION + `) Read more: https://github.com/jpillora/chisel - ` func main() { auth := flag.String("auth", "", "") verbose := flag.Bool("v", false, "") + version := flag.Bool("version", false, "") flag.Usage = func() { fmt.Fprintf(os.Stderr, help) } flag.Parse() + + if *version { + fmt.Println(VERSION) + os.Exit(1) + } + args := flag.Args() if len(args) < 2 { log.Fatalf("A server and least one remote is required") @@ -58,7 +68,7 @@ func main() { server := args[0] remotes := args[1:] - c, err := client.NewClient(*auth, server, remotes) + c, err := chiselclient.NewClient(*auth, server, remotes...) if err != nil { log.Fatal(err) } diff --git a/chisel_test.go b/chisel_test.go deleted file mode 100644 index 6da9a12..0000000 --- a/chisel_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package chisel - -import ( - "io/ioutil" - "net/http" - "testing" - "time" -) - -//all tests perform the follwing -// local proxy http server file server -// [2001] -> [2002] -> [2003] - -func TestSimple1(t *testing.T) { - - go fileserver(t) - go server(t) - time.Sleep(time.Second) - - go client(t) - time.Sleep(time.Second) - - resp, err := http.Get("http://localhost:2001/5000") - if err != nil { - t.Fatal(err) - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - - if len(b) != 5000 { - t.Fatal("5000 bytes expected") - } -} - -//============================ -// test helpers - -func server(t *testing.T) { - s, err := server.NewServer("foobar", "") - if err != nil { - t.Fatal(err) - } - - s.Info = true - s.Debug = true - - err = s.Start("0.0.0.0", "2002") - if err != nil { - t.Fatal(err) - } -} - -func client(t *testing.T) { - c, err := client.NewClient("foobar", "localhost:2002", "2001:2003") - if err != nil { - t.Fatal(err) - } - - c.Info = true - c.Debug = true - - err = c.Start() - if err != nil { - t.Fatal(err) - } -} - -func fileserver(t *testing.T) { - http.ListenAndServe("0.0.0.0:2003", nil) -} diff --git a/chiseld/main.go b/chiseld/main.go index 4ca2ded..f8f809d 100644 --- a/chiseld/main.go +++ b/chiseld/main.go @@ -9,31 +9,33 @@ import ( "github.com/jpillora/chisel/chiseld/server" ) -const help = ` +var VERSION string = "0.0.0-src" //set via ldflags + +var help = ` Usage: chiseld [options] Options: - --host defines the HTTP listening host – the - network interface (defaults to 0.0.0.0). You - may also set the HOST environment variable. - - --port defines the HTTP listening port (defaults - to 8080). You may also set the PORT environment + --host, Defines the HTTP listening host – the network interface + (defaults to 0.0.0.0). You may also set the HOST environment variable. - --auth specifies the exact authentication string - the client must provide to attain access. You - may also set the AUTH environment variable. + --port, Defines the HTTP listening port (defaults to 8080). You + may also set the PORT environment variable. - --proxy specifies the default proxy target to use - when chiseld receives a normal HTTP request. + --auth, Specifies the exact authentication string the client must + provide to attain access. You may also set the AUTH environment + variable. - -v enable verbose logging + --proxy, Specifies the default proxy target to use when chiseld + receives a normal HTTP request. + + -v, Enable verbose logging + + --version, Display version (` + VERSION + `) Read more: https://github.com/jpillora/chisel - ` func main() { @@ -43,12 +45,18 @@ func main() { authf := flag.String("auth", "", "") proxyf := flag.String("proxy", "", "") verbose := flag.Bool("v", false, "") + version := flag.Bool("version", false, "") flag.Usage = func() { fmt.Fprintf(os.Stderr, help) os.Exit(1) } flag.Parse() + if *version { + fmt.Println(VERSION) + os.Exit(1) + } + host := *hostf if host == "" { host = os.Getenv("HOST") @@ -70,7 +78,7 @@ func main() { auth = os.Getenv("AUTH") } - s, err := server.NewServer(auth, *proxyf) + s, err := chiselserver.NewServer(auth, *proxyf) if err != nil { log.Fatal(err) } diff --git a/chiseld/server/endpoint.go b/chiseld/server/endpoint.go index 45fb4ba..92cc7e4 100644 --- a/chiseld/server/endpoint.go +++ b/chiseld/server/endpoint.go @@ -1,4 +1,4 @@ -package server +package chiselserver import ( "net" diff --git a/chiseld/server/server.go b/chiseld/server/server.go index 671f637..a720d92 100644 --- a/chiseld/server/server.go +++ b/chiseld/server/server.go @@ -1,4 +1,4 @@ -package server +package chiselserver import ( "net/http" diff --git a/chiseld/server/websocket.go b/chiseld/server/websocket.go index 35c4b69..b04483b 100644 --- a/chiseld/server/websocket.go +++ b/chiseld/server/websocket.go @@ -1,4 +1,4 @@ -package server +package chiselserver import ( "encoding/binary" diff --git a/http_server.go b/http_server.go index ee4caa7..90cb0be 100644 --- a/http_server.go +++ b/http_server.go @@ -7,7 +7,8 @@ import ( "sync" ) -//HTTPServer is adds graceful shutdowns +//HTTPServer extends net/http Server and +//adds graceful shutdowns type HTTPServer struct { *http.Server listener net.Listener diff --git a/test/chisel_test.go b/test/chisel_test.go new file mode 100644 index 0000000..18a770a --- /dev/null +++ b/test/chisel_test.go @@ -0,0 +1,139 @@ +//chisel tests +//=================== +// (direct) +// .--------------->----------------. +// / chisel chisel \ +// request--->client:2001--->server:2002---->fileserver:3000 +// \ / +// '--> crowbar:4001--->crowbar:4002' +// client server +// +// benchmarks don't use testing.B, instead use +// go test -test.run=Bench +// +// tests use +// go test -test.run=Request + +package test + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + + "testing" + "time" +) + +const ( + B = 1 + KB = 1024 * B + MB = 1024 * KB + GB = 1024 * MB +) + +//test +func TestRequestChisel(t *testing.T) { + testTunnel("2001", 500, t) + testTunnel("2001", 50000, t) +} + +//benchmark +func TestBenchDirect(t *testing.T) { + benchSizes("3000", t) +} +func TestBenchChisel(t *testing.T) { + benchSizes("2001", t) +} +func TestBenchrowbar(t *testing.T) { + benchSizes("4001", t) +} + +func benchSizes(port string, t *testing.T) { + for size := 1; size < 100*MB; size *= 10 { + testTunnel(port, size, t) + } +} + +func testTunnel(port string, size int, t *testing.T) { + t0 := time.Now() + resp, err := requestFile(port, size) + if err != nil { + t.Fatal(err) + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + t1 := time.Now() + fmt.Printf(":%s => %d bytes in %s\n", port, size, t1.Sub(t0)) + if len(b) != size { + t.Fatalf("%d bytes expected, got %d", size, len(b)) + } +} + +//============================ + +//global setup +func TestMain(m *testing.M) { + + fs := makeFileServer() + go func() { + log.Fatal(fs.Wait()) + }() + + dir, _ := os.Getwd() + cd := exec.Command("crowbard", + `-listen`, "0.0.0.0:4002", + `-userfile`, path.Join(dir, "userfile")) + if err := cd.Start(); err != nil { + log.Fatal(err) + } + go func() { + log.Fatalf("crowbard: %s", cd.Wait()) + }() + + time.Sleep(100 * time.Millisecond) + + cf := exec.Command("crowbar-forward", + "-local=0.0.0.0:4001", + "-server=http://127.0.0.1:4002", + "-remote=127.0.0.1:3000", + "-username", "foo", + "-password", "bar") + if err := cf.Start(); err != nil { + log.Fatal(err) + } + + time.Sleep(100 * time.Millisecond) + + hd := exec.Command("chiseld", "--port", "2002", "--auth", "foobar") + if err := hd.Start(); err != nil { + log.Fatal(err) + } + hf := exec.Command("chisel-forward", + "--auth", "foobar", + "127.0.0.1:2002", + "2001:3000") + + if err := hf.Start(); err != nil { + log.Fatal(err) + } + + time.Sleep(100 * time.Millisecond) + + fmt.Println("Running!") + code := m.Run() + fmt.Println("Done") + + cd.Process.Kill() + cf.Process.Kill() + hd.Process.Kill() + hf.Process.Kill() + fs.Close() + + os.Exit(code) +} diff --git a/test/fileserver.go b/test/fileserver.go new file mode 100644 index 0000000..ccfb7bd --- /dev/null +++ b/test/fileserver.go @@ -0,0 +1,36 @@ +package test + +import ( + "net/http" + "strconv" + + "github.com/jpillora/chisel" +) + +func requestFile(port string, size int) (*http.Response, error) { + url := "http://127.0.0.1:" + port + "/" + strconv.Itoa(size) + // fmt.Println(url) + return http.Get(url) +} + +func makeFileServer() *chisel.HTTPServer { + bsize := 3 * MB + bytes := make([]byte, bsize) + //filling huge buffer + for i := 0; i < len(bytes); i++ { + bytes[i] = byte(i) + } + + s := chisel.NewHTTPServer() + s.SetKeepAlivesEnabled(false) + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rsize, _ := strconv.Atoi(r.URL.Path[1:]) + for rsize >= bsize { + w.Write(bytes) + rsize -= bsize + } + w.Write(bytes[:rsize]) + }) + s.GoListenAndServe("0.0.0.0:3000", handler) + return s +} diff --git a/test/userfile b/test/userfile new file mode 100644 index 0000000..ed3b07f --- /dev/null +++ b/test/userfile @@ -0,0 +1 @@ +foo:bar \ No newline at end of file