codemanki / cloudscraper Goto Github PK
View Code? Open in Web Editor NEW--DEPRECATED -- 🛑 🛑 Node.js library to bypass cloudflare's anti-ddos page
License: MIT License
--DEPRECATED -- 🛑 🛑 Node.js library to bypass cloudflare's anti-ddos page
License: MIT License
gang
@codemanki How could this be changed to use superagent instead of request? Thanks.
Hi,
previously the CF challenge was extracted with the following function:
var challenge = body.match(/S='([^']+)'/);
Now they made that variable random and added a random-name property on it for the challenge. Full example:
<script type="text/javascript">
--
| //<![CDATA[
| (function(){
| var a = function() {try{return !!window.addEventListener} catch(e) {return !1} },
| b = function(b, c) {a() ? document.addEventListener("DOMContentLoaded", b, c) : document.attachEvent("onreadystatechange", b)};
| b(function(){
| var a = document.getElementById('cf-content');a.style.display = 'block';
| setTimeout(function(){
| var s,t,o,p,b,r,e,a,k,i,n,g,f, bUNLJBB={"xqYrWXXfBO":+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(+[])+(+!![])+(!+[]+!![]))};
| t = document.createElement('div');
| t.innerHTML="<a href='/'>x</a>";
| t = t.firstChild.href;r = t.match(/https?:\/\//)[0];
| t = t.substr(r.length); t = t.substr(0,t.length-1);
| a = document.getElementById('jschl-answer');
| f = document.getElementById('challenge-form');
| ;bUNLJBB.xqYrWXXfBO-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]));bUNLJBB.xqYrWXXfBO+=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]));bUNLJBB.xqYrWXXfBO*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]))/+((+!![]+[])+(+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(+!![]));bUNLJBB.xqYrWXXfBO*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]));bUNLJBB.xqYrWXXfBO*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![])+(+!![])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]));bUNLJBB.xqYrWXXfBO-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+[])+(!+[]+!![]+!![])+(+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![])+(+!![])+(!+[]+!![]));bUNLJBB.xqYrWXXfBO*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]))/+((!+[]+!![]+[])+(+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]));bUNLJBB.xqYrWXXfBO*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(+!![])+(+[])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]));bUNLJBB.xqYrWXXfBO+=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]));a.value = +bUNLJBB.xqYrWXXfBO.toFixed(10) + t.length; '; 121'
| f.action += location.hash;
| f.submit();
| }, 4000);
| }, false);
| })();
| //]]>
| </script>
In the above example the challenge comes from the variable var bUNLJBB.xqYrWXXfBO
which returns a number. But as mentioned, the variable name is random
I can see this is possible to extract, but requires a lot more work. Has anybody been working on this already?
edit. I can see the extraction of the HTML form variables still works, and I do get past the CF challenge in some cases. However in some cases it fails, presumably due to the JavaScript above.
Could this be tweaked to work in the browser also?
Hello,
I have an issue with cloudscraper, i get an error code 1, when i try to get a page from hltv.org.
But i can get the source without any problem when i just use request.
Any ideas ?
Thanks for your job
is that possible to make a ts in the lib
Is it possible to send custom request headers?
I'm having subsequent failures when requesting to kissanime.com. Running this code gives me a null
error but a CloudFlare page
--------------
ERROR
--------------
null
--------------
BODY
--------------
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Please wait 5 seconds...</title>
<script type="text/javascript">
//<![CDATA[
(function(){
var a = function() {try{return !!window.addEventListener} catch(e) {return !1} },
b = function(b, c) {a() ? document.addEventListener("DOMContentLoaded", b, c) : document.attachEvent("onreadystatechange", b)};
b(function(){
var a = document.getElementById('cf-content');a.style.display = 'block';
setTimeout(function(){
var s,t,o,p,b,r,e,a,k,i,n,g,f, JIpeumZ={"TKWvJSqmT":+((!+[]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]))};
t = document.createElement('div');
t.innerHTML="<a href='/'>x</a>";
t = t.firstChild.href;r = t.match(/https?:\/\//)[0];
t = t.substr(r.length); t = t.substr(0,t.length-1);
a = document.getElementById('jschl-answer');
f = document.getElementById('challenge-form');
;JIpeumZ.TKWvJSqmT+=+((!+[]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]));JIpeumZ.TKWvJSqmT*=+((!+[]+!![]+[])+(+[]));JIpeumZ.TKWvJSqmT-=+((!+[]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]));JIpeumZ.TKWvJSqmT*=+((!+[]+!![]+!![]+[])+(+[]));a.value = parseInt(JIpeumZ.TKWvJSqmT, 10) + t.length; '; 121'
f.submit();
}, 4000);
}, false);
})();
//]]>
</script>
</head>
<body>
<div>
<div style="position: relative; height: 130px;">
<div style="max-width: 635px; margin: 0 auto; z-index: 1000; text-align: center;
position: relative; margin-top: 150px; width: 100%">
<img id="imgLogo" alt="jadopado" style="display: inline-block;" src="data:image/gif;charset=binary;base64,R0lGODlhZABkAPUaAPX19c/Pz+zs7Pr6+vHx8cXFxbe3t7KysuLi4srKysHBwdnZ2d3d3dTU1Ly8vOfn5/j4+Pz8/PLy8szMzN/f39zc3NbW1uXl5enp6dPT0+/v7/39/fT09O3t7erq6ujo6Pf39////4+Pj7i4uODg4HBwcK2trYWFhcLCwqOjo3p6epmZmevr62ZmZgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQFAAAaACwAAAAAZABkAAAG/8CQcEgsGo/IpHLJbDqf0Kh0Sq1ar9isdsvter/gsHhMLpvP6LR6zW6731WAQACAuwWKg/6gENjVCHuCBwh/ZwKDg36GZAWJggWMYwOPgwOSYYiVe4uYUhACFwIQR5qbB51EAAgLCHWeRBALE7S0FRFGp3pGAwmDBa+eAAG1xQGkRL6bCUUABo8GBLAZxdULRQTP0NJEjpUOngLV4xJFCNqCBoVEppXrjBXj1RdGw4IBwUMLusySs/K1riEBkK+It02R/AGsRYFKAF0JGSFYSAsDFQa6GmACQHECMikAdHGTNBEgvSr7KmmERUHeuyoNHgWANURCvAkBKpTLIiCBtv8EqWgKHUq0qNGjXQA0cFQgQNAxAgIUMFCgQUE0gRKtJBMz0cszWR9tDdP10VcyA9A9usol5CYDl8BmFFPWHZqUCMUcrCTQDN5KEb/sfdS3zN9Hgb0MTlSYzANdDMRgPPUgTZ5KcMWk3aRATbZKlcc83raGwGAHT8EIcPBrZOkFsEOjeQB7gWukuHPr3l1lwBwBcbX4nhOc5oCHghoUpzKg7oEAyxk5e6QgOpQBlwcZYGsne6LEURbv6YwpbKXbTwjoOgsH+abGTw4nmilJPCQq9veAh5NfT78iA9C2wAPWKZOXJM4lElkRDKhlwIJETLbJWH+0s1YR7g1C3xBubZL/GhwGalXEaKAVkeAe/0kygHgbDsHaJuBgiJh1hiCQnQJndbjJVTaOx15uFj7yIW9JBJnIkEQiocsBSUaR4XxNXqdWOjRGiQQBLwriAHpWLjEAA94owECVXZZp5plopqnmmmy26eabcBpBgggmFMHCCi20cIIFRIggQgpDoOCnnygMYYIIJMwpAp9z1hnCnHkiCocFLYhQxAktpGBCCSWwIASleb4yQgsqiFBCCyMIIUILFoAKKKWWhoDpCniqMGmlRJCgpxAmoCrEqJgWGsKoqepagqqsgnosrI/uGsKpfLrB7BCjOkrpCsiikOmvvsraAgkhrNpqC6dOwGy1QpBg2QF3Z0zLbaohMAsAqfPaOmy34oabbKUqpHBut3+4ey+8zE7QArYqtOApscjyKS6svRpsKcODCtuGwAzHi2uvqabQQqEZ5/twpbp6PLGvIiQM78W4Utsts5jmmSe2IX+r77iWqnDqyY5mzHKsnx787ryVioDpsQzPe+zNzPaKK6Un8AowG5SWUDEAp6IwQcIWaBu1EHmSMKoII2AKKNO4shBpCESPgMKpK1Mtc56pvi1znR6fre8Io8qcwisjx4pprNrm6XHckrTKbhYAWABunJBHLvnklFcOeRAAIfkEBQoACgAsHAASAA0ADQAABDdQSQXQQmBOUI73yTARxmcameKYbKEgbCwkMbt09Vfg+RE0PQ+DEDxkgLnFJFBraBQzjyEhmEQAACH5BAUAAAEALBUACwA8ADwAAAaSwJBwSCwajxEIJHJsOp9QJ8RzqV48kKh2GwVYvxcAd0wOQcDgbHn9FKC/ArbcuHmDN/O82f5V69lnfFV+f2uCVYVzHYIdiXIDD3YPA45yAJFgD2KVchEaXxpMnHoRoqOnqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbXUUEAIfkEBQAAAwAsLAALAA4ADgAABT/gIIqQJEFjOkjW5E6WpApvPQmjZNvysOy1xSACtEV0xZcpqRwwXaIKsyICBIoBwEhwtQVwKQilRkGpRhBzKgQAIfkEBQAACwAsLAALAA4ADgAABDtwSQmEADMvoY4/iqAhX3kgk2Ca4lKsZbEMsDmo9Wfl+sJ7kgQvISEYagbCBHE0GVAZQKAUwGgmAGsmAgAh+QQFAAABACw9ABIADQANAAAFOWAgBhEgAdE4bt3lukIqRs9rP4Mo2PwXQLygKcjr7IgvgQaZ5DBdhMjzkpMwCSMCUaP6dWqXDmQUAgAh+QQFAAADACw9ABIADQANAAAFOeAgDpBwCdA4RtXkuksqQsFrB4C42Hw2SLygKcir7IivBQWZxDBdCMhzkrswESMEkaL6VWqTimQUAgAh+QQFAAAKACw9ABIADQANAAAENVBJBdBCYM6RjvdFJgHGZxqEVJisowhsbMVsstJfEeA5w3sNwO+QWvAakwYtoHklSoeEYBIBACH5BAUAAAEALEQAIwANAA0AAAY7wIAQohE8BISIcEm4OJ8XABNKlUao1MemiYUCBF2oBBx2jstmDvoC2jzKHiEo/BgsIR+qwL6US/58QkEAIfkEBQAAAwAsRAAjAA0ADQAABTfgIErUEiwIJK7I5L6TwMK0DNF0ELU4LCw92AUYdA2LRgxyookEihaRJhgArCQZ2sK6kl6+XFEIACH5BAUAAAkALEQAIwANAA0AAAQzMEkRiikNyN2O/wfCgaQIkKQxdCiIFC24wLE31zaDH89g1ArJI2YgbAQOUsG4ES6eTEkEACH5BAUAAAEALD0AMwANAA0AAAU6YCAGG2Ru47gRV9tqqLh5bv1EolDvXjDsQIAEuNPoiC5BB5nkMFuEyPMCCbCQnZGGKIiJALSWBzAKAQAh+QQFAAADACw9ADMADQANAAAFOuAgDpFkRuMYIVPbUqgYWW4dQOJS79YA7EDBBbij6IiuRQWZxDBbCMhzIhmwkJURhbiIiQS0lkUwCgEAIfkEBQAADgAsPQAzAA0ADQAABDbQSTeEHXOOdnoPmDQoXmkAUlGuikOsMLLAa6DSXpHgOcN3DcDvIHBwcIlJgFYISRCkjgIxiQAAIfkEBQAAAAAsLAA6AA4ADQAABkVAgBCwAUkkoM1wyXlcnpcHZwnQQK8XzRCExYKEn+71A4iIsRHI+QpRr58QwPsptK61RKf4oRQOwlgfA1QbHAJPHhx9AEEAIfkEBQAAAgAsLAA6AA4ADQAABUCgIAqRdl1aNK5YML1TgK0CBd8TNWo4romZ3i0jgAhxEMnxJlEuXxLB8yWyLXUklzCgEgGCuAyAFsEsXhZMVxACACH5BAUAAAgALCwAOgAOAA0AAAQ8EEk03lpvzM3M+YfBbEgAnkcwPSj6SE57Ogggo4Bwn4K+fwLE7yMx7VQUj8ygkRBiKAeBNGAUPgpGExEBACH5BAUAAAEALBwAMwANAA0AAAU6YCAGG2Ru47hpV9sSqBg9bu3FXq0LAaD/A9avJhEMawLjsdUhLFscyPMSCXSWBNFGqdOkermWBzAKAQAh+QQFAAADACwcADMADQANAAAFOuAgDpFkRuMYUVPbIqgIBW5txVatL4Og/wDWr3ZZDGsL47FVQSxbGMlzAhlUlghRRKmjpHq5lkUwCgEAIfkEBQAADgAsHAAzAA0ADQAABDbQSTeEHXOOcHpvmAQYXqmESqkWDqK+BPeWSzGXhX13SbN3DMHvAHAkdg3JQKcKZFqpjgIxiQAAIfkEBQAAAQAsFQAjAA0ADQAABj3AgDBCEFwEGohwCbg4nxcCE0qVbh5UaqSZhRIkXagAHHaOy2YQ+sIJeMqPTWCA7YKWAyP0o1wKBxKBd0tBACH5BAUAAAMALBUAIwANAA0AAAU54CBCyDItlCSuwuS+E8LCtBwFNA21OYxcPdgCGHQNi0YNcoIZWIqByACA62lWABMso1qJAJfwdRUCACH5BAUAAAkALBUAIwANAA0AAAQ1MEnQyilByI2O/0fDgaQ4GCQJdCnYLC1YwLE31/aDH0yi1IZBgoBqPTYEC8ih2UgIi+hxEwEAIfkEBQAAAQAsHAASAA0ADQAABTtgIAYRIAHROEbC5brdNg7Paz9p8Nm8EAC8IKQT5ElaxZcAmbxoCE0XZxC9pKBJyUhTJKgCQ9ejAxmFAAA7"><br />
</div>
</div>
<div>
<div style="text-align: center">
Please wait 5 seconds...<br />Make sure to enable cookies and javascript.<br />This site does not work with "Mini browsers" (e.g. UC mini, Opera mini...)
</div>
<div style="margin: 0 auto; visibility: hidden">
<div class="cf-browser-verification cf-im-under-attack">
<noscript><h1 data-translate="turn_on_js" style="color:#bd2426;">Please turn JavaScript on and reload the page.</h1></noscript>
<div id="cf-content" style="display:none">
<div>
<div class="bubbles"></div>
<div class="bubbles"></div>
<div class="bubbles"></div>
</div>
<h1><span data-translate="checking_browser">Checking your browser before accessing</span> kissanime.to.</h1>
<p data-translate="process_is_automatic">This process is automatic. Your browser will redirect to your requested content shortly.</p>
<p data-translate="allow_5_secs">Please allow up to 5 seconds…</p>
</div>
<form id="challenge-form" action="/cdn-cgi/l/chk_jschl" method="get">
<input type="hidden" name="jschl_vc" value="b5ef2df50c21a08804dd3238e6669288"/>
<input type="hidden" name="pass" value="1480319486.85-CJpLFmT3Vp"/>
<input type="hidden" id="jschl-answer" name="jschl_answer"/>
Init
</form>
</div>
</div>
</div>
<div style="padding: 20px;">
</div>
</div>
</body>
</html>
I'm not sure myself if CloudFlare is throttling me. I was able to get this library to work for a single time an hour ago, but never had any another success again. Will try again tonight if I can get this to work.
My rss parser worked just fine until met this link
https://zooqle.com/шерлок-sherlock-04x01-из-03-2017-hdtv-1080i-от-exkinoray-первый-vk0la.html
And cloudscrapper can't get body of this page. I suppose it have something to do with russian letters in url.
Hi, is there any way to implement proxies into this?
I am trying to bypass Bitgo cloudflare which is running on my mannual Server (http://www.rubitcurrency.com/login).
Once I click /login it takes time but nothing happens. For immediate testing purpose I like to know where to paste code that you have written in your Readme.
If i try to scraping this url http://vcrypt.net/shield/HSXWXJKcvcJTgm4AAGhG9db4LFAyMO6dlnaQCpm65PSHoDhUM1tZWG_ppl_LgwpDPRbC it returns error 2 - 1008.
I'm tring to understand which is the meaning of this error but the page linked into the wiki of this project is wrong. Could you please help me?
Thanks
Is there a way of using Promises with cloudscraper ?
I am having issues when trying to retrieve a website that is in "I'm under attack mode" (http://kissanime.com)
With a very basic cloudscraper.get, no error is returned however the body of the response is the Cloudflare challenge page.
I have noticed this is a sister site to the website currently being investigated in Issue #1, so that might hold some relevance?
Interesting thing to note is that this issue does not occur when using cloudflare-scraper
Is there the possibility to specify the User Agent when made GET?
If no, can it be implemented?
Thanks
Hi,
First of all: thanks for a great job! Saved me heaps of time :)
In my project, I need a persistent jar, so I have a small suggestion.
In the first couple of lines, you create a private jar. How about changing this slightly:
var defaultJar = requestModule.jar();
var request = requestModule.defaults({jar: defaultJar}), // Cookies should be enabled
Then, as the first line in function setCookieAndReload:
var jar = options.jar || defaultJar;
It now uses the (shared) jar from the options, or - if that is not provided - the default (private) jar.
Works like a charm!
Thanks again,
Goswinus
Re-using the request makes it possible to re-use the request agent (in some cases) and preserves header information that may be lost. This should be a non-breaking change. Request has smart option handling so it's quite easy to achieve this.
Consider the following code example.
requestModule('https://example.com', function(error, response, body) {
var options = response.request;
console.log(options.uri, options.headers, options.agent);
options.uri = 'https://example.com/path';
requestModule(options, callback);
});
Also, if we want to just reinitialize(redirect) the request, it would simplify the code base as you don't have to pass around the callback or options. You'd only need to pass the response object around. challengesToSolve
could be entirely removed as request
's redirect options can be used instead.
Today I noticed that the scraper stopped working for me although it worked before.
This is the url that I tried to get: https://www.spigotmc.org/resources/5583/download?version=140619
Source:
cloudscraper.request({method: 'GET', url:url, encoding: null}, function(err, response, body) {});
And that's the body, while error is empty:
<!DOCTYPE html>
<html ng-app="downloadApp" ng-controller="parentController">
<head>
<title ng-bind="pageTitle">Download</title>
<script type="text/javascript">
//<![CDATA[
try{if (!window.CloudFlare) {var CloudFlare=[{verbose:0,p:0,byc:0,owlid:"cf",bag2:1,mirage2:0,oracle:0,paths:{cloudflare:"/cdn-cgi/nexp/dok3v=1613a3a185/"},atok:"452c95e5d0ef3672607ab953ca2cc8ae",petok:"4bd31b8e0a7167e6fb0beca53a8864b1d2371cd7-1487097634-1800",zone:"inventivetalent.org",rocket:"0",apps:{"ga_key":{"ua":"UA-43843691-3","ga_bs":"2"}}}];!function(a,b){a=document.createElement("script"),b=document.getElementsByTagName("script")[0],a.async=!0,a.src="//ajax.cloudflare.com/cdn-cgi/nexp/dok3v=f2befc48d1/cloudflare.min.js",b.parentNode.insertBefore(a,b)}()}}catch(e){};
//]]>
</script>
<link id="favicon" rel="shortcut icon" type="image/png" href="/favicon_full.png"/>
<link href="/css/bootstrap.min.css" rel="stylesheet">
<meta name=viewport content="width=device-width, initial-scale=1">
<base href="/">
<style>.centered{position:fixed;top:50%;left:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%);}</style>
<script type="text/javascript">
/* <![CDATA[ */
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-43843691-3']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
(function(b){(function(a){"__CF"in b&&"DJS"in b.__CF?b.__CF.DJS.push(a):"addEventListener"in b?b.addEventListener("load",a,!1):b.attachEvent("onload",a)})(function(){"FB"in b&&"Event"in FB&&"subscribe"in FB.Event&&(FB.Event.subscribe("edge.create",function(a){_gaq.push(["_trackSocial","facebook","like",a])}),FB.Event.subscribe("edge.remove",function(a){_gaq.push(["_trackSocial","facebook","unlike",a])}),FB.Event.subscribe("message.send",function(a){_gaq.push(["_trackSocial","facebook","send",a])}));"twttr"in b&&"events"in twttr&&"bind"in twttr.events&&twttr.events.bind("tweet",function(a){if(a){var b;if(a.target&&a.target.nodeName=="IFRAME")a:{if(a=a.target.src){a=a.split("#")[0].match(/[^?=&]+=([^&]*)?/g);b=0;for(var c;c=a[b];++b)if(c.indexOf("url")===0){b=unescape(c.split("=")[1]);break a}}b=void 0}_gaq.push(["_trackSocial","twitter","tweet",b])}})})})(window);
/* ]]> */
</script>
</head>
<body>
<div class="container">
<div ng-view>
<div class="centered">
<h1>Loading Download...</h1>
</div>
</div>
</div>
<script src="https://code.jquery.com/jquery-3.1.1.min.js" integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.6.1/angular.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.4.8/angular-route.js" integrity="sha384-k+Qp/8rZxoiiYGVjOBiZwkEp5yv6clgl2EmwNaE1oUMlfmEYgCWazxf4CyxfZiWG" crossorigin="anonymous"></script>
<script src="js/script.js"></script>
</body>
</html>
I am going to crawl a webpage which I use casperJS, how can I use the cloudscraper to bypass the cloudfare challenge with casperjs, any example?
First, thanks for the project!
Would it be possible to integrate incapsula support? It looks like they have similar antiddos strategies and there is also a python library available for bypassing it: https://github.com/ziplokk1/incapsula-cracker-py3
Cloudscraper failed for this page: https://steamdb.info/freepackages/
Can you fix it?
Website is https://mcsniper.co/ - they have a custom cloudflare page which I believe may be the root cause of the issue.
Code snippet:
cloudscraper.request({
method: "GET",
url: "https://mcsniper.co/",
proxy: "http://[REDACTED]",
headers: {
"User-Agent": "[REDACTED]",
},
timeout: (30 * 1000),
}
Error occurred during evaluation: Unexpected token } when crawling a site with CloudFare protection. Seem they has changed their js :(
wow, wonderful! it help me so much! thanks!
I'm seeing the error message "Error occurred during evaluation: Unexpected string" while trying to scrape this: http://zolmax.com/feed
Link to example: https://repl.it/repls/ExpensiveMortifiedDevices
Running it prints the js challenge page instead of expected page
hi
everything is in the title
i added :
followAllRedirects: true
to all your performRequest functions and everything is right :)
(don t have time to fork / make PR now, sorry)
Allow us to solve and submit captchas for pages that requires one. For example, you could configure https://anti-captcha.com to solve them.
cloudscraper.post("/api", {
location: "Österreich",
ip: "192.168.0.12",
});
But it shows on Server Side
{ location: "Österreich",
ip: '192.168.0.1' }
I'm modifying the module to get the cf_clearance cookie, but the original web response doesn't have it, it have other cookies but not this one, which is the one needed to get the web without bypassing it again.
In which stage could i get this one?
For what reason add tough-cookie in dependencies if you do not currently use? That makes unnecessary dependencies are installed. Additionally this version presents vulnearibilidades
Hi,
Thanks for your sharing,
I have another case, can you help me solve it ?
Thanks
I don't think you are using the method
variable here https://github.com/codemanki/cloudscraper/blob/master/index.js#L65
On Crunchyroll, they have activated the "email protection" from cloudflare and when I try to scrape some of their webpages I get "[email protected]" instead of the expected text.
Example: http://www.crunchyroll.com/the-idolmster-cinderella-girls-theater#
On that webpage the "DOLM@STER" is seen as an email by cloudflare (that mean their regexp is fabulously wrong) and it screw my scraping because I'm not expecting ton of the javascript they insert for that "protection".
It would be nice if cloudscraper could detect these insertion and parse them the same as it does for the browser detection.
i write a "cloudscraper" by superagent module, but not success. i think superagent cookie jar module have problem.
the same code, i change to use request module. it succeed.
i still not found the problem on superagent. crazy..
This will boost the npmjs package rank.
https://itnext.io/increasing-an-npm-packages-search-score-fb557f859300
The current coding style is SemiStandard, no changes required.
When i'm using proxy for HTTPS requests after long time sometimes happen this:
node_modules/cloudscraper/index.js:201
callback(error, response, body.toString(options.realEncoding));
^
TypeError: Cannot read property 'toString' of undefined
at giveResults (node_modules/cloudscraper/index.js:201:35)
at Request._callback (node_modules/cloudscraper/index.js:186:7)
at self.callback (node_modules/request/request.js:200:22)
at emitOne (events.js:77:13)
at Request.emit (events.js:169:7)
at Request.onRequestError (node_modules/request/request.js:831:8)
at emitOne (events.js:77:13)
at ClientRequest.emit (events.js:169:7)
at Socket.socketErrorListener (_http_client.js:264:9)
at emitOne (events.js:77:13)
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
process.env.HTTP_PROXY = 'http://138.0.172.145:80'; //this proxy sometimes works..
I've seen it work with similar sites, but it doesn't seem to work with http://www.moneyunder30.com/feed
Hi!
What I'm asking is, I'm making a small REST API for a website, and I was wondering, is it possible to do a POST to a login page, and save the session cookie to that website, and keep being logged in for the rest of the session?
Hope it makes sense!
Hi, I'm trying to cache the requests but sending the 'Cache-Control' header doesn't seem to have any effect. Here's the code:
cloudscraper.request(
{
method: "GET",
url: link,
encoding: null,
headers: {
"content-type": "image/jpeg",
"Cache-Control": "max-age:31556926"
}
},
makeRequest(options, function(error, response, body) {
var validationError;
var stringBody = body.toString('utf8');
if (validationError = checkForErrors(error, stringBody)) {
return callback(validationError, body, response);
}
// If body contains specified string, solve challenge
if (stringBody.indexOf('a = document.getElementById(\'jschl-answer\');') !== -1) {
setTimeout(function() {
return solveChallenge(response, stringBody, options, callback);
}, Timeout);
} else {
// All is good
giveResults(options, error, response, body, callback);
}
});
}
function checkForErrors(error, body) {
var match;
// Pure request error (bad connection, wrong url, etc)
if(error) {
return { errorType: 0, error: error };
}
// Finding captcha
if (body.indexOf('why_captcha') !== -1 || /recaptcha/i.test(body)) {
return { errorType: 1 };
}
// trying to find '<span class="cf-error-code">1006</span>'
match = body.match(/<\w+\s+class="cf-error-code">(.*)<\/\w+>/i);
if (match) {
return { errorType: 2, error: parseInt(match[1]) };
}
return false;
}
body.toString() is called unsafely. Some request errors cause body to be undefined.
@codemanki :
Hi, I wanted to know if it works on react-native.
I tried to install it but it does not seem to work, from problems with addictions.
So I wanted to know is it possible to make it work on react-native?
v3rmillion.net
as subject :)
if possible, can you expose the stream interface from request
?
I'd like to see support for request defaults such as jar: true
, which remembers cookies for subsequent requests.
is it possible to submit login form and then being authenticated. for example login via github.com/login and then get a page that need login for example,https://github.com/settings/profile
This library really should extend request the same way that request-promise
does. It'd probably close all of the currently open issues. I'll work on a bit later but here is the gist of it.
The example below shows how to support request
and request-promise
. Simply have them both as peer dependencies (not installed automatically) and users can continue to use request
as they normally would.
I'll update to show response handling but I think it's all pretty straight forward if you look at the code for request
and request-promise
.
// Snippet removed (See edit history or referenced PR)
with the "Request" object, I could get images doing
var requestSettings = {
url: img_url,
method: 'GET',
encoding: null
};
request.get(requestSettings, function (error, response, body) { ... });
I'd like to do the same with cloud scraper, otherwise it takes the file but it's not recognized as image (probably for the wrong encoding)
A declarative, efficient, and flexible JavaScript library for building user interfaces.
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google ❤️ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.