ProjectDiscovery httpX: reconnaissance

Basic Usage #

Probing URLs and Hosts #

bash

# Probe a single URL
httpx -u https://example.com

# Probe multiple URLs
httpx -u https://example.com,https://projectdiscovery.io

# Probe from a list of URLs/hosts
httpx -l hosts.txt

# Probe from STDIN
cat hosts.txt|httpx

Output Options #

bash

# Save results to a file
httpx -l hosts.txt -o results.txt

# Output in JSON format
httpx -l hosts.txt -json -o results.json

# Output in CSV format
httpx -l hosts.txt -csv -o results.csv

# Silent mode (only URLs)
httpx -l hosts.txt -silent

Basic Filtering #

bash

# Filter by status code
httpx -l hosts.txt -status-code 200

# Filter by content length
httpx -l hosts.txt -content-length 100

# Match specific title
httpx -l hosts.txt -title "Dashboard"

# Match specific technology
httpx -l hosts.txt -tech wordpress

Advanced Usage #

Port Scanning #

bash

# Scan default ports (80, 443)
httpx -l hosts.txt

# Scan specific ports
httpx -l hosts.txt -ports 80,443,8080,8443

# Scan top 100 ports
httpx -l hosts.txt -ports top-100

# Scan all ports
httpx -l hosts.txt -ports all

Path Probing #

bash

# Probe specific paths
httpx -l hosts.txt -path /api/v1,/admin,/login

# Probe from a file containing paths
httpx -l hosts.txt -path-file paths.txt

# Automatically add trailing slash
httpx -l hosts.txt -path /api -add-slash

Protocol Options #

bash

# Force HTTPS
httpx -l hosts.txt -https

# Probe both HTTP and HTTPS
httpx -l hosts.txt -probe

# Skip HTTPS verification
httpx -l hosts.txt -no-verify

Request Customization #

bash

# Set custom headers
httpx -l hosts.txt -H "User-Agent: Mozilla/5.0" -H "Cookie: session=123456"

# Set HTTP method
httpx -l hosts.txt -method POST

# Set request body
httpx -l hosts.txt -method POST -body "username=admin&password=admin"

# Set content type
httpx -l hosts.txt -method POST -H "Content-Type: application/json" -body '\\\\{"username":"admin","password":"admin"\\\\}'

Response Filtering #

bash

# Match response containing specific string
httpx -l hosts.txt -match-string "admin"

# Match response using regex
httpx -l hosts.txt -match-regex "admin.*panel"

# Filter response not containing string
httpx -l hosts.txt -filter-string "not found"

# Filter response using regex
httpx -l hosts.txt -filter-regex "error|not found"

Screenshot Capture #

bash

# Capture screenshots
httpx -l hosts.txt -screenshot

# Specify screenshot output directory
httpx -l hosts.txt -screenshot -screenshot-output screenshots/

# Set screenshot timeout
httpx -l hosts.txt -screenshot -screenshot-timeout 20

Technology Detection #

bash

# Detect web technologies
httpx -l hosts.txt -tech-detect

# Output only specific technologies
httpx -l hosts.txt -tech-detect -match-tech wordpress,nginx

Performance Optimization #

Concurrency and Rate Limiting #

bash

# Set concurrency (default: 50)
httpx -l hosts.txt -concurrency 100

# Set rate limit
httpx -l hosts.txt -rate-limit 200

# Set request timeout
httpx -l hosts.txt -timeout 10

Retry and Delay Options #

bash

# Set maximum retries
httpx -l hosts.txt -retries 3

# Set delay between requests
httpx -l hosts.txt -delay 2s

# Set random delay
httpx -l hosts.txt -random-agent

Optimization for Large Scans #

bash

# Use stream mode for large inputs
httpx -l large-hosts.txt -stream

# Skip default ports probing
httpx -l hosts.txt -no-default-ports

# Skip failed host probes
httpx -l hosts.txt -skip-host-error

Integration with Other Tools #

Pipeline with Subfinder #

bash

# Find subdomains and probe them
subfinder -d example.com|httpx

# Find subdomains, probe them, and check for specific paths
subfinder -d example.com|httpx -path /api,/admin -status-code 200

Pipeline with Nuclei #

bash

# Find active hosts and scan for vulnerabilities
httpx -l hosts.txt -silent|nuclei -t cves/

# Find hosts with specific tech and scan for related vulnerabilities
httpx -l hosts.txt -tech-detect -match-tech wordpress -silent|nuclei -t wordpress/

Pipeline with Naabu #

bash

# Scan ports and probe HTTP services
naabu -host example.com -top-ports 1000 -silent|httpx

# Scan ports, probe HTTP services, and check for vulnerabilities
naabu -host example.com -top-ports 1000 -silent|httpx -silent|nuclei -t cves/

Output Customization #

Custom Output Format #

bash

# Define custom output format
httpx -l hosts.txt -o results.txt -silent -format "\\\\{\\\\{.StatusCode\\\\}\\\\} \\\\{\\\\{.URL\\\\}\\\\} \\\\{\\\\{.Title\\\\}\\\\}"

# Include specific fields in output
httpx -l hosts.txt -include-response-time -include-chain -include-cdn

Response Extraction #

bash

# Extract title
httpx -l hosts.txt -title

# Extract favicon hash
httpx -l hosts.txt -favicon

# Extract response headers
httpx -l hosts.txt -response-header

# Extract TLS information
httpx -l hosts.txt -tls-grab

Response Storage #

bash

# Store response bodies
httpx -l hosts.txt -store-response

# Specify response storage directory
httpx -l hosts.txt -store-response -store-response-dir responses/

# Store chain responses
httpx -l hosts.txt -store-chain

Advanced Filtering #

Status Code Filtering #

bash

# Match specific status codes
httpx -l hosts.txt -status-code 200,301,302

# Filter out specific status codes
httpx -l hosts.txt -exclude-status-code 404,403

Content Filtering #

bash

# Filter by content length
httpx -l hosts.txt -content-length 100

# Match content length range
httpx -l hosts.txt -content-length-lt 1000 -content-length-gt 100

# Filter by content type
httpx -l hosts.txt -content-type "text/html"

Header Filtering #

bash

# Match specific header
httpx -l hosts.txt -match-header "Server: nginx"

# Filter by header presence
httpx -l hosts.txt -include-headers "Server,Content-Type"

Proxy and Network Options #

bash

# Use HTTP proxy
httpx -l hosts.txt -proxy http://127.0.0.1:8080

# Use SOCKS5 proxy
httpx -l hosts.txt -proxy socks5://127.0.0.1:1080

# Follow redirects
httpx -l hosts.txt -follow-redirects

# Follow redirects with max depth
httpx -l hosts.txt -follow-redirects -follow-max-redirects 5

# Follow host redirects
httpx -l hosts.txt -follow-host-redirects

Miscellaneous Features #

CRLF Injection Detection #

bash

# Check for CRLF injection
httpx -l hosts.txt -crlf

CORS Misconfiguration Check #

bash

# Check for CORS misconfigurations
httpx -l hosts.txt -cors

IP Geolocation #

bash

# Include IP geolocation information
httpx -l hosts.txt -location

Web Cache Detection #

bash

# Check for web cache
httpx -l hosts.txt -web-cache

Virtual Host Discovery #

bash

# Probe for virtual hosts
httpx -l hosts.txt -vhost

# Specify vhost wordlist
httpx -l hosts.txt -vhost -vhost-wordlist vhosts.txt

Troubleshooting #

Common Issues #

Connection Timeouts

bash

# Increase timeout
httpx -l hosts.txt -timeout 15
   
# Increase retries
httpx -l hosts.txt -retries 3

Rate Limiting by Target

bash

# Reduce concurrency
httpx -l hosts.txt -concurrency 10
    
# Add delay between requests
httpx -l hosts.txt -delay 2s

Memory Issues

bash

# Use stream mode for large inputs
httpx -l large-hosts.txt -stream
    
# Reduce concurrency
httpx -l hosts.txt -concurrency 25

TLS/SSL Errors

bash

# Skip TLS verification
httpx -l hosts.txt -no-verify

Debugging #

bash

# Enable verbose mode
httpx -l hosts.txt -verbose

# Show request and response details
httpx -l hosts.txt -debug

# Show only failed requests
httpx -l hosts.txt -debug-req -debug-resp -silent

Configuration #

Configuration File #

HTTPX uses a configuration file located at $HOME/.config/httpx/config.yaml. You can customize various settings in this file:

bash

# Example configuration file
concurrency: 50
timeout: 5
retries: 2
rate-limit: 150
verbose: false
silent: false
output: httpx_output.txt

Environment Variables #

bash

# Set HTTPX configuration via environment variables
export HTTPX_CONCURRENCY=50
export HTTPX_TIMEOUT=5
export HTTPX_RETRIES=2
export HTTPX_RATE_LIMIT=150

Reference #

Command Line Options #

Flag	Description
`-u, -target`	Target URL/host to probe
`-l, -list`	File containing list of URLs/hosts to probe
`-o, -output`	File to write output to
`-json`	Write output in JSON format
`-csv`	Write output in CSV format
`-silent`	Show only URLs/hosts in output
`-verbose`	Show verbose output
`-debug`	Show request/response details
`-version`	Show HTTPX version
`-ports`	Ports to probe (default: 80,443)
`-path`	Path(s) to probe
`-method`	HTTP method to use
`-status-code`	Filter by status code
`-title`	Filter by title
`-content-length`	Filter by content length
`-tech-detect`	Detect web technologies
`-follow-redirects`	Follow HTTP redirects
`-no-verify`	Skip TLS verification
`-H, -header`	Custom header to add to all requests
`-match-string`	Match response containing string
`-match-regex`	Match response using regex
`-filter-string`	Filter response not containing string
`-filter-regex`	Filter response not matching regex
`-screenshot`	Take screenshots of websites
`-concurrency`	Number of concurrent requests
`-rate-limit`	Maximum number of requests per second
`-timeout`	Timeout in seconds for HTTP requests
`-retries`	Number of retries for failed requests
`-delay`	Delay between requests
`-proxy`	HTTP/SOCKS5 proxy to use

Output Fields #

Field	Description
`url`	Target URL
`input`	Original input
`scheme`	URL scheme (http/https)
`host`	Target host
`port`	Target port
`path`	URL path
`status_code`	HTTP status code
`title`	Page title
`content_type`	Content type header
`content_length`	Content length
`response_time`	Response time in seconds
`technologies`	Detected technologies
`server`	Server header
`webserver`	Detected web server
`ip`	Target IP address
`cdn`	CDN information
`favicon`	Favicon hash
`tls`	TLS information
`location`	Redirect location
`vhost`	Virtual host information