admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2309 from GDATTACKER-RESEARCHER/dev

Browse Source 
Improve error handling in htmlToText function
This commit is contained in:
Mzack9999
2025-11-18 16:55:40 +04:00
committed by GitHub
parent 45d84765da 05c6364d90
commit c6948ed01a
4 changed files with 26 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/httpx/httpx.go
+1 -1
View File
@@ -1,6 +1,7 @@
package httpx
import (
"context"
"crypto/tls"
"fmt"
"io"
@@ -25,7 +26,6 @@ import (
pdhttputil "github.com/projectdiscovery/utils/http"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
"golang.org/x/net/context"
"golang.org/x/net/http2"
)
common/pagetypeclassifier/pagetypeclassifier.go
+8 -11
View File
@@ -14,26 +14,23 @@ type PageTypeClassifier struct {
classifier *naive_bayes.NaiveBayesClassifier
}
func New() *PageTypeClassifier {
func New() (*PageTypeClassifier, error) {
classifier, err := naive_bayes.NewClassifierFromFileData(classifierData)
if err != nil {
panic(err)
return nil, err
}
return &PageTypeClassifier{classifier: classifier}
return &PageTypeClassifier{classifier: classifier}, nil
}
func (n *PageTypeClassifier) Classify(html string) string {
text := htmlToText(html)
if text == "" {
text, err := htmlToText(html)
if err != nil || text == "" {
return "other"
}
return n.classifier.Classify(text)
}
func htmlToText(html string) string {
text, err := htmltomarkdown.ConvertString(html)
if err != nil {
panic(err)
}
return text
// htmlToText safely converts HTML to text and protects against panics from Go's HTML parser.
func htmlToText(html string) (string, error) {
return htmltomarkdown.ConvertString(html)
}
common/pagetypeclassifier/pagetypeclassifier_test.go
+12 -7
View File
@@ -3,19 +3,22 @@ package pagetypeclassifier
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPageTypeClassifier(t *testing.T) {
t.Run("test creation of new PageTypeClassifier", func(t *testing.T) {
epc := New()
assert.NotNil(t, epc)
epc, err := New()
require.NoError(t, err)
require.NotNil(t, epc)
})
t.Run("test classification non error page text", func(t *testing.T) {
epc := New()
assert.Equal(t, "nonerror", epc.Classify(`<!DOCTYPE html>
epc, err := New()
require.NoError(t, err)
require.NotNil(t, epc)
require.Equal(t, "nonerror", epc.Classify(`<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
@@ -30,8 +33,10 @@ func TestPageTypeClassifier(t *testing.T) {
})
t.Run("test classification on error page text", func(t *testing.T) {
epc := New()
assert.Equal(t, "error", epc.Classify(`<!DOCTYPE html>
epc, err := New()
require.NoError(t, err)
require.NotNil(t, epc)
require.Equal(t, "error", epc.Classify(`<!DOCTYPE html>
<html>
<head>
<title>Error 403: Forbidden</title>
runner/runner.go
+5 -1
View File
@@ -385,7 +385,11 @@ func New(options *Options) (*Runner, error) {
}
runner.simHashes = gcache.New[uint64, struct{}](1000).ARC().Build()
runner.pageTypeClassifier = pagetypeclassifier.New()
pageTypeClassifier, err := pagetypeclassifier.New()
if err != nil {
return nil, err
}
runner.pageTypeClassifier = pageTypeClassifier
if options.HttpApiEndpoint != "" {
apiServer := NewServer(options.HttpApiEndpoint, options)