admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into added-whitelist-blacklist

Browse Source
This commit is contained in:
Mzack9999
2021-04-29 17:51:32 +02:00
committed by GitHub
parent 21b92bd904 e9bb05db10
commit 0c78db6218
10 changed files with 69 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yaml
+1 -1
View File
@@ -12,7 +12,7 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v2.4.0
uses: golangci/golangci-lint-action@v2.5.2
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v1.31
Dockerfile
+3
View File
@@ -3,6 +3,9 @@ RUN apk add --no-cache git
RUN GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx
FROM alpine:latest
RUN apk -U upgrade --no-cache \
&& apk add --no-cache bind-tools ca-certificates
COPY --from=builder /go/bin/httpx /usr/local/bin/
ENTRYPOINT ["httpx"]
Makefile
+14
View File
@@ -0,0 +1,14 @@
# Go parameters
GOCMD=go
GOBUILD=$(GOCMD) build
GOMOD=$(GOCMD) mod
GOTEST=$(GOCMD) test
GOGET=$(GOCMD) get
all: build
build:
$(GOBUILD) -v -ldflags="-extldflags=-static" -o "httpx" cmd/httpx/httpx.go
test:
$(GOTEST) -v ./...
tidy:
$(GOMOD) tidy
README.md
+4
View File
@@ -112,13 +112,16 @@ This will display help for the tool. Here are all the switches it supports.
| ports | Ports ranges to probe (nmap syntax: eg 1,2-10,11) | httpx -ports 80,443,100-200 |
| title | Prints title of page if available | httpx -title |
| path | Request path/file | httpx -path /api |
| paths | Request list of paths from file | httpx -paths paths.txt |
| content-length | Prints content length in the output | httpx -content-length |
| ml | Match content length in the output | httpx -content-length -ml 125 |
| fl | Filter content length in the output | httpx -content-length -fl 0,43 |
| status-code | Prints status code in the output | httpx -status-code |
| mc | Match status code in the output | httpx -status-code -mc 200,302 |
| fc | Filter status code in the output | httpx -status-code -fc 404,500 |
| tech-detect | Perform wappalyzer based technology detection | httpx -tech-detect |
| tls-probe | Send HTTP probes on the extracted TLS domains | httpx -tls-probe |
| tls-grab | Perform TLS data grabbing | httpx -tls-grab |
| content-type | Prints content-type | httpx -content-type |
| location | Prints location header | httpx -location |
| csp-probe | Send HTTP probes on the extracted CSP domains | httpx -csp-probe |
@@ -128,6 +131,7 @@ This will display help for the tool. Here are all the switches it supports.
| unsafe | Send raw requests skipping golang normalization | httpx -unsafe |
| request | File containing raw request to process | httpx -request |
| retries | Number of retries | httpx -retries |
| random-agent | Use randomly selected HTTP User-Agent header value | httpx -random-agent |
| silent | Prints only results in the output | httpx -silent |
| stats | Prints statistic every 5 seconds | httpx -stats |
| timeout | Timeout in seconds (default 5) | httpx -timeout 10 |
common/httputilz/httputilz.go
+1 -1
View File
@@ -27,7 +27,7 @@ func DumpRequest(req *retryablehttp.Request) (string, error) {
// DumpResponseHeadersAndRaw returns http headers and response as strings
func DumpResponseHeadersAndRaw(resp *http.Response) (header, response string, err error) {
// httputil.DumpResponse does not work with websockets
if resp.StatusCode == http.StatusContinue {
if resp.StatusCode >= http.StatusContinue || resp.StatusCode <= http.StatusEarlyHints {
raw := resp.Status + "\n"
for h, v := range resp.Header {
raw += fmt.Sprintf("%s: %s\n", h, v)
go.mod
+2 -2
View File
@@ -1,6 +1,6 @@
module github.com/projectdiscovery/httpx
go 1.14
go 1.16
require (
github.com/corpix/uarand v0.1.1
@@ -20,6 +20,6 @@ require (
github.com/projectdiscovery/wappalyzergo v0.0.2
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.3.0
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
golang.org/x/text v0.3.6
)
go.sum
+10 -2
View File
@@ -1,3 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
@@ -45,6 +46,11 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
@@ -57,7 +63,6 @@ github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
@@ -118,7 +123,7 @@ github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8
github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.11 h1:jyzTass/CD3MgaK4pQSXJzwb91ksVYocwiE0AQ1ytEo=
github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
ojectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/projectdiscovery/wappalyzergo v0.0.2 h1:pBqpp04ngQjFr1LPqHcJ7qaiL0AOVY19IGmSjg5/+DI=
github.com/projectdiscovery/wappalyzergo v0.0.2/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
@@ -231,6 +236,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
internal/runner/banner.go
+2 -2
View File
@@ -8,11 +8,11 @@ const banner = `
/ __ \/ __/ __/ __ \| /
/ / / / /_/ /_/ /_/ / |
/_/ /_/\__/\__/ .___/_/|_|
/_/ v1.0.4-dev
/_/ v1.0.5
`
// Version is the current version of httpx
const Version = `v1.0.4-dev`
const Version = `v1.0.5`
// showBanner is used to show the banner to the user
func showBanner() {
internal/runner/options.go
+5
View File
@@ -50,6 +50,8 @@ type scanOptions struct {
PreferHTTPS bool
NoFallback bool
TechDetect bool
OutputExtractRegex string
extractRegex *regexp.Regexp
}
func (s *scanOptions) Clone() *scanOptions {
@@ -82,6 +84,7 @@ func (s *scanOptions) Clone() *scanOptions {
PreferHTTPS: s.PreferHTTPS,
NoFallback: s.NoFallback,
TechDetect: s.TechDetect,
OutputExtractRegex: s.OutputExtractRegex,
}
}
@@ -156,6 +159,7 @@ type Options struct {
RandomAgent bool
Deny customlist.CustomList
Allow customlist.CustomList
OutputExtractRegex string
}
// ParseOptions parses the command line options for application
@@ -221,6 +225,7 @@ func ParseOptions() *Options {
flag.BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value")
flag.Var(&options.Allow, "allow", "Whitelist ip/cidr")
flag.Var(&options.Deny, "deny", "Blacklist ip/cidr")
flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex")
flag.Parse()
internal/runner/runner.go
+27 -8
View File
@@ -2,7 +2,6 @@ package runner
import (
"bufio"
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
@@ -13,6 +12,7 @@ import (
"net/url"
"os"
"path"
"regexp"
"strconv"
"strings"
"time"
@@ -173,6 +173,11 @@ func New(options *Options) (*Runner, error) {
scanopts.OutputResponseTime = options.OutputResponseTime
scanopts.NoFallback = options.NoFallback
scanopts.TechDetect = options.TechDetect
if options.OutputExtractRegex != "" {
if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil {
return nil, err
}
}
// output verb if more than one is specified
if len(scanopts.Methods) > 1 && !options.Silent {
@@ -522,6 +527,12 @@ retry:
URL := fmt.Sprintf("%s://%s", protocol, domain)
if port > 0 {
URL = fmt.Sprintf("%s://%s:%d", protocol, domain, port)
} else {
domainParse := strings.Split(domain, ":")
domain = domainParse[0]
if len(domainParse) > 1 {
port, _ = strconv.Atoi(domainParse[1])
}
}
if !scanopts.Unsafe {
@@ -542,9 +553,6 @@ retry:
req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody))
}
// Create a copy on the fly of the request body - ignore errors
bodyBytes, _ := req.BodyBytes()
req.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
requestDump, err := httputil.DumpRequestOut(req.Request, true)
if err != nil {
return Result{URL: URL, err: err}
@@ -654,9 +662,13 @@ retry:
builder.WriteString(fmt.Sprintf(" [%s]", serverHeader))
}
var serverResponseRaw = ""
var serverResponseRaw string
var request string
var responseHeader string
if scanopts.ResponseInStdout {
serverResponseRaw = string(resp.Data)
request = string(requestDump)
responseHeader = resp.RawHeaders
}
// check for virtual host
@@ -690,7 +702,6 @@ retry:
builder.WriteString(" [http2]")
}
}
ip := hp.Dialer.GetDialedIP(domain)
if scanopts.OutputIP {
builder.WriteString(fmt.Sprintf(" [%s]", ip))
@@ -743,6 +754,14 @@ retry:
}
}
// extract regex
if scanopts.extractRegex != nil {
matches := scanopts.extractRegex.FindAllString(string(resp.Data), -1)
if len(matches) > 0 {
builder.WriteString(" [" + strings.Join(matches, ",") + "]")
}
}
// store responses in directory
if scanopts.StoreResponse {
domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)
@@ -792,8 +811,8 @@ retry:
return Result{
Timestamp: time.Now(),
Request: string(requestDump),
ResponseHeader: resp.RawHeaders,
Request: request,
ResponseHeader: responseHeader,
Scheme: parsed.Scheme,
Port: finalPort,
Path: finalPath,