JavaScript Rendering

If the page you intend to scrape necessitates the execution of JavaScript in order to dynamically load all the necessary data you may enable JavaScript rendering without the need of using a headless browser.

Receive rendered page content in HTML

curl -k -v -x unblock.smartproxy.com:60000 \
-U "USERNAME:PASSWORD" "https://ip.smartproxy.com/" \
-H "X-SU-Headless: html"

import requests

proxies = {
    'http': 'http://USERNAME:[email protected]:60000',
    'https': 'http://USERNAME:[email protected]:60000'
}

headers= {
   "X-SU-Headless": "html"
}

response = requests.request(
    'GET',
    'https://ip.smartproxy.com/',
    verify=False,
    proxies=proxies,
    headers=headers
)

print(response.text)

import fetch from 'node-fetch';
import createHttpsProxyAgent from 'https-proxy-agent'

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const agent = createHttpsProxyAgent(
  `http://${username}:${password}@unblock.smartproxy.com:60000`
);

process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;

const headers = {
 	'X-SU-Headless': 'html'
}

const response = await fetch('https://ip.smartproxy.com/', {
  method: 'get',
  headers: headers,
  agent: agent
});

console.log(await response.text());

Receive rendered page content as PNG

The response will contain raw bytes of an image that can be saved in PNG format

curl -k -v -x unblock.smartproxy.com:60000 \
-U "USERNAME:PASSWORD" "https://ip.smartproxy.com/" \
-H "X-SU-Headless: png"

import requests

proxies = {
    'http': 'http://USERNAME:[email protected]:60000',
    'https': 'http://USERNAME:[email protected]:60000'
}

headers= {
   "X-SU-Headless": "png"
}

response = requests.request(
    'GET',
    'https://ip.smartproxy.com/',
    verify=False,
    proxies=proxies,
    headers=headers
)

print(response.text)

import fetch from 'node-fetch';
import createHttpsProxyAgent from 'https-proxy-agent'

const username = 'YOUR_USERNAME';
const password = 'YOUR_PASSWORD';

const agent = createHttpsProxyAgent(
  `http://${username}:${password}@unblock.smartproxy.com:60000`
);

process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;

const headers = {
 	'X-SU-Headless': 'png'
}

const response = await fetch('https://ip.smartproxy.com/', {
  method: 'get',
  headers: headers,
  agent: agent
});

console.log(await response.text());