My goal is to use a proxy to scrape certain websites for some data.
My code is below:
import requests
from bs4 import BeautifulSoup
proxies = {'socks4': 'http://162.144.66.22:16603'}
r = requests.get("http://google.com", proxies=proxies)
soup = BeautifulSoup(r.content, "html.parser")
print(r)
print(soup)
The problem is that it appears the proxy may not be being used. As when I change the proxies dictionary to a random number, it still returns status 200 and the html. Any ideas why this is happening? Is it using the proxy or not?
Results:
<Response [200]>
<!DOCTYPE html>
<html itemscope="" itemtype="http://schema.org/WebPage" lang="en"><head><meta content="Search the world's information, including webpages, images, videos and more. Google has many special features to help you find exactly what you're looking for." name="description"/><meta content="noodp" name="robots"/><meta content="text/html; charset=utf-8" http-equiv="Content-Type"/><meta content="/images/branding/googleg/1x/googleg_standard_color_128dp.png" itemprop="image"/><title>Google</title><script nonce="ogEjPeXsr0ScuC6g9Ol/1g==">(function(){window.google={kEI:'N-GTYYidKsXKtQaYh6a4Bg',kEXPI:'0,1302536,56873,6059,206,4804,2316,383,246,5,1354,4012,924,315,1122515,1197760,625,328882,51224,16114,28684,17572,4859,1361,9291,3022,17586,4020,978,13228,3847,4192,6430,1142,20680,919,1832,3249,889,704,1279,2212,530,149,1103,840,1983,214,4100,3514,606,2023,1777,520,14670,3227,2845,7,5599,6755,5096,11625,2771,1926,906,2,941,2614,12710,432,3,346,1244,1,5445,148,11323,991,1661,4,1528,2304,7039,74,1983,2627,2014,11501,3824,1012,2038,2658,7356,31,894,1,10518,2215,1592,713,675,1457,15351,1435,5830,2527,4094,17,3121,6,908,3,3541,1,14261,450,1813,283,912,5992,12346,2332,769,8,1,2987,2,9212,627,4,32,4,4143,1244,4,683,442,342,255,55,4,2811,123,1557,743,2351,3502,1576,3,7826,49,1009,1160,6190,2,508,2380,2719,3708,2,5566,4893,4127,1,8,3022,4696,4568,2577,3132,278,271,1107,5525,85,1,3779,3,1514,2,2920,3147,545,3751,1039,438,5672,516,414,46,3273,1720,2072,701,69,2495,2183,2,1037,44,217,1156,285,1253,41,833,341,2,1096,330,3,2,20,3,2,121,2204,341,225,1161,352,4,99,122,373,201,1007,61,792,239,20,155,4,7,52,31,33,335,111,2,2,1,332,613,95,367,852,731,550,1587,114,449,81,1371,434,103,3390,28,142,263,147,60,4,49,584,18,1258,401,282,127,192,209,328,4,52,29,5,507,245,532,2,684,1234,780,404,2,98,369,7,2,2,2,3,3,1,7,3,1,2,3,2,2,8,7,187,852,117,283,12,635,183,74,131,732,6,2,708,54,853,2,252,1170,499,5532101,100,346,313,16,1802560,4193980,2800696,882,444,1,2,80,1,1796,1,9,2553,1,748,141,795,563,1,4265,1,1,2,1331,4142,2609,155,17,13,72,139,4,2,20,2,169,13,19,46,5,39,96,548,29,2,2,1,2,1,2,2,7,4,1,2,2,2,2,2,2,353,513,186,1,1,158,3,2,2,2,2,2,4,2,3,3,269,551,782,49,8,16,4,9,5,20,45,1,30,4,2,16,2,17,20733141,3220020,4041352,338,3,2414,1491,9,1435,157,1360,291,115,3900,2886,501,655,810367',kBL:'4bzV'};google.sn='webhp';google.kHL='en';})();(function(){
var f=this||self;var h,k=[];function l(a){for(var b;a&&(!a.getAttribute||!(b=a.getAttribute("eid")));)a=a.parentNode;return b||h}function m(a){for(var b=null;a&&(!a.getAttribute||!(b=a.getAttribute("leid")));)a=a.parentNode;return b}
function n(a,b,c,d,g){var e="";c||-1!==b.search("&ei=")||(e="&ei="+l(d),-1===b.search("&lei=")&&(d=m(d))&&(e+="&lei="+d));d="";!c&&f._cshid&&-1===b.search("&cshid=")&&"slh"!==a&&(d="&cshid="+f._cshid);c=c||"/"+(g||"gen_204")+"?atyp=i&ct="+a+"&cad="+b+e+"&zx="+Date.now()+d;/^http:/i.test(c)&&"https:"===window.location.protocol&&(google.ml&&google.ml(Error("a"),!1,{src:c,glmm:1}),c="");return c};h=google.kEI;google.getEI=l;google.getLEI=m;google.ml=function(){return null};google.log=function(a,b,c,d,g){if(c=n(a,b,c,d,g)){a=new Image;var e=k.length;k[e]=a;a.onerror=a.onload=a.onabort=function(){delete k[e]};a.src=c}};google.logUrl=n;}).call(this);(function(){
google.y={};google.sy=[];google.x=function(a,b){if(a)var c=a.id;else{do c=Math.random();while(google.y[c])}google.y[c]=[a,b];return!1};google.sx=function(a){google.sy.push(a)};google.lm=[];google.plm=function(a){google.lm.push.apply(google.lm,a)};google.lq=[];google.load=function(a,b,c){google.lq.push([[a],b,c])};google.loadAll=function(a,b){google.lq.push([a,b])};google.bx=!1;google.lx=function(){};}).call(this);google.f={};(function(){
document.documentElement.addEventListener("submit",function(b){var a;if(a=b.target){var c=a.getAttribute("data-submitfalse");a="1"===c||"q"===c&&!a.elements.q.value?!0:!1}else a=!1;a&&(b.preventDefault(),b.stopPropagation())},!0);document.documentElement.addEventListener("click",function(b){var a;a:{for(a=b.target;a&&a!==document.documentElement;a=a.parentElement)if("A"===a.tagName){a="1"===a.getAttribute("data-nohref");break a}a=!1}a&&b.preventDefault()},!0);}).call(this);</script><style>#gbar,#guser{font-size:13px;padding-top:1px !important;}#gbar{height:22px}#guser{padding-bottom:7px !important;text-align:right}.gbh,.gbd{border-top:1px solid #c9d7f1;font-size:1px}.gbh{height:0;position:absolute;top:24px;width:100%}#media all{.gb1{height:22px;margin-right:.5em;vertical-align:top}#gbar{float:left}}a.gb1,a.gb4{text-decoration:underline !important}a.gb1,a.gb4{color:#00c !important}.gbi .gb4{color:#dd8e27 !important}.gbf .gb4{color:#900 !important}
</style><style>body,td,a,p,.h{font-family:arial,sans-serif}body{margin:0;overflow-y:scroll}#gog{padding:3px 8px 0}td{line-height:.8em}.gac_m td{line-height:17px}form{margin-bottom:20px}.h{color:#1558d6}em{font-weight:bold;font-style:normal}.lst{height:25px;width:496px}.gsfi,.lst{font:18px arial,sans-serif}.gsfs{font:17px arial,sans-serif}.ds{display:inline-box;display:inline-block;margin:3px 0 4px;margin-left:4px}input{font-family:inherit}body{background:#fff;color:#000}a{color:#4b11a8;text-decoration:none}a:hover,a:active{text-decoration:underline}.fl a{color:#1558d6}a:visited{color:#4b11a8}.sblc{padding-top:5px}.sblc a{display:block;margin:2px 0;margin-left:13px;font-size:11px}.lsbb{background:#f8f9fa;border:solid 1px;border-color:#dadce0 #70757a #70757a #dadce0;height:30px}.lsbb{display:block}#WqQANb a{display:inline-block;margin:0 12px}.lsb{background:url(/images/nav_logo229.png) 0 -261px repeat-x;border:none;color:#000;cursor:pointer;height:30px;margin:0;outline:0;font:15px arial,sans-serif;vertical-align:top}.lsb:active{background:#dadce0}.lst:focus{outline:none}</style><script nonce="ogEjPeXsr0ScuC6g9Ol/1g==">(function(){window.google.erd={sp:'hp',jsr:0,bv:1476,de:true};
var f=this||self;var g,h,k=null!==(g=f.mei)&&void 0!==g?g:1,l=null!==(h=f.sdo)&&void 0!==h?h:!0,n=0,p,q=google.erd,t=q.jsr;google.ml=function(a,b,e,m,d){d=void 0===d?2:d;b&&(p=a&&a.message);if(google.dl)return google.dl(a,d,e),null;if(0>t){window.console&&console.error(a,e);if(-2===t)throw a;b=!1}else b=!a||!a.message||"Error loading script"===a.message||n>=k&&!m?!1:!0;if(!b)return null;n++;e=e||{};b=encodeURIComponent;var c="/gen_204?atyp=i&ei="+b(google.kEI);google.kEXPI&&(c+="&jexpid="+b(google.kEXPI));c+="&srcpg="+b(google.sn)+"&jsr="+b(q.jsr)+"&bver="+b(q.bv)+("&jsel="+d);for(var r in e)c+="&",c+=b(r),c+="=",c+=
b(e[r]);c=c+"&emsg="+b(a.name+": "+a.message);c=c+"&jsst="+b(a.stack||"N/A");12288<=c.length&&(c=c.substr(0,12288));a=c;m||google.log(0,"",a);return a};window.onerror=function(a,b,e,m,d){p!==a&&(a=d instanceof Error?d:Error(a),google.ml(a,!1,void 0,!1,"SyntaxError"===a.name||"SyntaxError"===a.message.substring(0,11)?2:0));p=null;l&&n>=k&&(window.onerror=null)};})();</script></head><body bgcolor="#fff"><script nonce="ogEjPeXsr0ScuC6g9Ol/1g==">(function(){var src='/images/nav_logo229.png';var iesg=false;document.body.onload = function(){window.n && window.n();if (document.images){new Image().src=src;}
if (!iesg){document.f&&document.f.q.focus();document.gbqf&&document.gbqf.q.focus();}
}
})();</script><div id="mngb"><div id="gbar"><nobr><b class="gb1">Search</b> <a class="gb1" href="http://www.google.com/imghp?hl=en&tab=wi">Images</a> <a class="gb1" href="http://maps.google.com/maps?hl=en&tab=wl">Maps</a> <a class="gb1" href="https://play.google.com/?hl=en&tab=w8">Play</a> <a class="gb1" href="http://www.youtube.com/?gl=US&tab=w1">YouTube</a> <a class="gb1" href="https://news.google.com/?tab=wn">News</a> <a class="gb1" href="https://mail.google.com/mail/?tab=wm">Gmail</a> <a class="gb1" href="https://drive.google.com/?tab=wo">Drive</a> <a class="gb1" href="https://www.google.com/intl/en/about/products?tab=wh" style="text-decoration:none"><u>More</u> »</a></nobr></div><div id="guser" width="100%"><nobr><span class="gbi" id="gbn"></span><span class="gbf" id="gbf"></span><span id="gbe"></span><a class="gb4" href="http://www.google.com/history/optout?hl=en">Web History</a> | <a class="gb4" href="/preferences?hl=en">Settings</a> | <a class="gb4" href="https://accounts.google.com/ServiceLogin?hl=en&passive=true&continue=http://www.google.com/&ec=GAZAAQ" id="gb_70" target="_top">Sign in</a></nobr></div><div class="gbh" style="left:0"></div><div class="gbh" style="right:0"></div></div><center><br clear="all" id="lgpd"/><div id="lga"><img alt="Google" height="92" id="hplogo" src="/images/branding/googlelogo/1x/googlelogo_white_background_color_272x92dp.png" style="padding:28px 0 14px" width="272"/><br/><br/></div><form action="/search" name="f"><table cellpadding="0" cellspacing="0"><tr valign="top"><td width="25%"> </td><td align="center" nowrap=""><input name="ie" type="hidden" value="ISO-8859-1"/><input name="hl" type="hidden" value="en"/><input name="source" type="hidden" value="hp"/><input name="biw" type="hidden"/><input name="bih" type="hidden"/><div class="ds" style="height:32px;margin:4px 0"><input autocomplete="off" class="lst" maxlength="2048" name="q" size="57" style="margin:0;padding:5px 8px 0 6px;vertical-align:top;color:#000" title="Google Search" value=""/></div><br style="line-height:0"/><span class="ds"><span class="lsbb"><input class="lsb" name="btnG" type="submit" value="Google Search"/></span></span><span class="ds"><span class="lsbb"><input class="lsb" id="tsuid1" name="btnI" type="submit" value="I'm Feeling Lucky"/><script nonce="ogEjPeXsr0ScuC6g9Ol/1g==">(function(){var id='tsuid1';document.getElementById(id).onclick = function(){if (this.form.q.value){this.checked = 1;if (this.form.iflsig)this.form.iflsig.disabled = false;}
else top.location='/doodles/';};})();</script><input name="iflsig" type="hidden" value="ALs-wAMAAAAAYZPvR3XeA_2hvvUXKtup1vz1LZFktom0"/></span></span></td><td align="left" class="fl sblc" nowrap="" width="25%">Advanced search</td></tr></table><input id="gbv" name="gbv" type="hidden" value="1"/><script nonce="ogEjPeXsr0ScuC6g9Ol/1g==">(function(){
var a,b="1";if(document&&document.getElementById)if("undefined"!=typeof XMLHttpRequest)b="2";else if("undefined"!=typeof ActiveXObject){var c,d,e=["MSXML2.XMLHTTP.6.0","MSXML2.XMLHTTP.3.0","MSXML2.XMLHTTP","Microsoft.XMLHTTP"];for(c=0;d=e[c++];)try{new ActiveXObject(d),b="2"}catch(h){}}a=b;if("2"==a&&-1==location.search.indexOf("&gbv=2")){var f=google.gbvu,g=document.getElementById("gbv");g&&(g.value=a);f&&window.setTimeout(function(){location.href=f},0)};}).call(this);</script></form><div id="gac_scont"></div><div style="font-size:83%;min-height:3.5em"><br/></div><span id="footer"><div style="font-size:10pt"><div id="WqQANb" style="margin:19px auto;text-align:center">Advertising ProgramsBusiness SolutionsAbout Google</div></div><p style="font-size:8pt;color:#70757a">© 2021 - Privacy - Terms</p></span></center><script nonce="ogEjPeXsr0ScuC6g9Ol/1g==">(function(){window.google.cdo={height:757,width:1440};(function(){
var a=window.innerWidth,b=window.innerHeight;if(!a||!b){var c=window.document,d="CSS1Compat"==c.compatMode?c.documentElement:c.body;a=d.clientWidth;b=d.clientHeight}a&&b&&(a!=google.cdo.width||b!=google.cdo.height)&&google.log("","","/client_204?&atyp=i&biw="+a+"&bih="+b+"&ei="+google.kEI);}).call(this);})();</script> <script nonce="ogEjPeXsr0ScuC6g9Ol/1g==">(function(){google.xjs={ck:'',cs:'',excm:[]};})();</script> <script nonce="ogEjPeXsr0ScuC6g9Ol/1g==">(function(){var u='/xjs/_/js/k\x3dxjs.hp.en_US.0nQ8X6LkAxI.O/am\x3dACcAVg/d\x3d1/ed\x3d1/esmo\x3d1/rs\x3dACT90oFK4c5-TQYA7GFcdnA53Ix1Arix4g/m\x3dsb_he,d';
var e=this||self,f=function(a){return a};var g;var l=function(a,b){this.g=b===h?a:""};l.prototype.toString=function(){return this.g+""};var h={};
function m(){var a=u;google.lx=function(){n(a);google.lx=function(){}};google.bx||google.lx()}
function n(a){google.timers&&google.timers.load&&google.tick&&google.tick("load","xjsls");var b=document;var c="SCRIPT";"application/xhtml+xml"===b.contentType&&(c=c.toLowerCase());c=b.createElement(c);if(void 0===g){b=null;var k=e.trustedTypes;if(k&&k.createPolicy){try{b=k.createPolicy("goog#html",{createHTML:f,createScript:f,createScriptURL:f})}catch(p){e.console&&e.console.error(p.message)}g=b}else g=b}a=(b=g)?b.createScriptURL(a):a;a=new l(a,h);c.src=a instanceof l&&a.constructor===l?a.g:"type_error:TrustedResourceUrl";var d;a=(c.ownerDocument&&c.ownerDocument.defaultView||window).document;(d=(b=null===(d=a.querySelector)||void 0===d?void 0:d.call(a,"script[nonce]"))?b.nonce||b.getAttribute("nonce")||"":"")&&c.setAttribute("nonce",d);document.body.appendChild(c);google.psa=!0};setTimeout(function(){m()},0);})();(function(){window.google.xjsu='/xjs/_/js/k\x3dxjs.hp.en_US.0nQ8X6LkAxI.O/am\x3dACcAVg/d\x3d1/ed\x3d1/esmo\x3d1/rs\x3dACT90oFK4c5-TQYA7GFcdnA53Ix1Arix4g/m\x3dsb_he,d';})();function _DumpException(e){throw e;}
function _F_installCss(c){}
(function(){google.jl={attn:false,blt:'none',chnk:0,dw:false,dwu:true,emtn:0,end:0,ine:false,lls:'default',pdt:0,rep:0,snet:true,strt:0,ubm:false,uwp:true};})();(function(){var pmc='{\x22d\x22:{},\x22sb_he\x22:{\x22agen\x22:true,\x22cgen\x22:true,\x22client\x22:\x22heirloom-hp\x22,\x22dh\x22:true,\x22dhqt\x22:true,\x22ds\x22:\x22\x22,\x22ffql\x22:\x22en\x22,\x22fl\x22:true,\x22host\x22:\x22google.com\x22,\x22isbh\x22:28,\x22jsonp\x22:true,\x22msgs\x22:{\x22cibl\x22:\x22Clear Search\x22,\x22dym\x22:\x22Did you mean:\x22,\x22lcky\x22:\x22I\\u0026#39;m Feeling Lucky\x22,\x22lml\x22:\x22Learn more\x22,\x22oskt\x22:\x22Input tools\x22,\x22psrc\x22:\x22This search was removed from your \\u003Ca href\x3d\\\x22/history\\\x22\\u003EWeb History\\u003C/a\\u003E\x22,\x22psrl\x22:\x22Remove\x22,\x22sbit\x22:\x22Search by image\x22,\x22srch\x22:\x22Google Search\x22},\x22ovr\x22:{},\x22pq\x22:\x22\x22,\x22refpd\x22:true,\x22rfs\x22:[],\x22sbas\x22:\x220 3px 8px 0 rgba(0,0,0,0.2),0 0 0 1px rgba(0,0,0,0.08)\x22,\x22sbpl\x22:16,\x22sbpr\x22:16,\x22scd\x22:10,\x22stok\x22:\x229dnURCIg88JrBfNLoHlile_Fe8M\x22,\x22uhde\x22:false}}';google.pmc=JSON.parse(pmc);})();</script> </body></html>
Process finished with exit code 0
I am trying to login to a website and read the contents of its webpage(which comes up after I login) using Python. I have got the login part but I'm not able to read the webpage contents. I have tried using urlopen(url). But it again opens the login page. Might be due to authentication being required.
I need help on reading the webpage which opens after logging in.
Try "requests" package with your url
[~]$ python
Python 2.7.5+ (default, Sep 17 2013, 17:31:54)
[GCC 4.8.1] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>>
>>> import requests
>>>
>>> r = requests.get('http://www.google.com')
>>>
>>> r.status_code
200
>>> r.content
'<!doctype html><html itemscope="" itemtype="http://schema.org/WebPage" lang="en"><head><meta content="Search the world\'s information, including webpages, images, videos and more. Google has many special features to help you find exactly what you\'re looking for." name="description"><meta content="noodp" name="robots"><meta content="/logos/doodles/2015/eiji-tsuburayas-114th-birthday-4809204506296320.2-hp.jpg" itemprop="image"><meta content="Make a movie with Eiji Tsuburaya! #GoogleDoodle" property="og:description"><meta content="http://www.google.com/logos/doodles/2015/eiji-tsuburayas-114th-birthday-4809204506296320.2-hp.jpg" property="og:image"><meta content="356" property="og:image:width"><meta content="200" property="og:image:height"><title>Google</title><script>(function(){window.google={kEI:\'QuybVeO7Osy4sAXDxIKQCg\',kEXPI:\'18168,3700062,3700306,4020727,4029815,4032235,4032500,4032678,4033307,4033344,4034631,4034882,4036363,4036366,4036470,4036848,4036948,4037333,4037457,4037855,4037921,4037960,4038777,4039047,4039280,4039379,4039386,4039403,4039706,4039878,4040028,4040117,4040135,4040240,8300096,8300200,8300202,8500394,8501295,8501406,8501489,10200083,10201088,10201270,10201302,10201304\',authuser:0,kscs:\'c9c918f0_10\'};google.kHL=\'en\';})();(function(){google.lc=[];google.li=0;google.getEI=function(a){for(var b;a&&(!a.getAttribute||!(b=a.getAttribute("eid")));)a=a.parentNode;return b||google.kEI};google.getLEI=function(a){for(var b=null;a&&(!a.getAttribute||!(b=a.getAttribute("leid")));)a=a.parentNode;return b};google.https=function(){return"https:"==window.location.protocol};google.ml=function(){return null};google.time=function(){return(new Date).getTime()};google.log=function(a,b,d,e,g){a=google.logUrl(a,b,d,e,g);if(""!=a){b=new Image;var c=google.lc,f=google.li;c[f]=b;b.onerror=b.onload=b.onabort=function(){delete c[f]};window.google&&window.google.vel&&window.google.vel.lu&&window.google.vel.lu(a);b.src=a;google.li=f+1}};google.logUrl=function(a,b,d,e,g){var c="",f=google.ls||"";if(!d&&-1==b.search("&ei=")){var h=google.getEI(e),c="&ei="+h;-1==b.search("&lei=")&&((e=google.getLEI(e))?c+="&lei="+e:h!=google.kEI&&(c+="&lei="+google.kEI))}a=d||"/"+(g||"gen_204")+"?atyp=i&ct="+a+"&cad="+b+c+f+"&zx="+google.time();/^http:/i.test(a)&&google.https()&&(google.ml(Error("a"),!1,{src:a,glmm:1}),a="");return a};google.y={};google.x=function(a,b){google.y[a.id]=[a,b];return!1};google.load=function(a,b,d){google.x({id:a+k++},function(){google.load(a,b,d)})};var k=0;})();google.kCSI={};var _gjwl=location;function _gjuc(){var a=_gjwl.href.indexOf("#");if(0<=a&&(a=_gjwl.href.substring(a),0<a.indexOf("&q=")||0<=a.indexOf("#q="))&&(a=a.substring(1),-1==a.indexOf("#"))){for(var d=0;d<a.length;){var b=d;"&"==a.charAt(b)&&++b;var c=a.indexOf("&",b);-1==c&&(c=a.length);b=a.substring(b,c);if(0==b.indexOf("fp="))a=a.substring(0,d)+a.substring(c,a.length),c=d;else if("cad=h"==b)return 0;d=c}_gjwl.href="/search?"+a+"&cad=h";return 1}return 0}\nfunction _gjh(){!_gjuc()&&window.google&&google.x&&google.x({id:"GJH"},function(){google.nav&&google.nav.gjh&&google.nav.gjh()})};window._gjh&&_gjh();</script><style>#gbar,#guser{font-size:13px;padding-top:1px !important;}#gbar{height:22px}#guser{padding-bottom:7px !important;text-align:right}.gbh,.gbd{border-top:1px solid #c9d7f1;font-size:1px}.gbh{height:0;position:absolute;top:24px;width:100%}#media all{.gb1{height:22px;margin-right:.5em;vertical-align:top}#gbar{float:left}}a.gb1,a.gb4{text-decoration:underline !important}a.gb1,a.gb4{color:#00c !important}.gbi .gb4{color:#dd8e27 !important}.gbf .gb4{color:#900 !important}\n</style><style>body,td,a,p,.h{font-family:arial,sans-serif}body{margin:0;overflow-y:scroll}#gog{padding:3px 8px 0}td{line-height:.8em}.gac_m td{line-height:17px}form{margin-bottom:20px}.h{color:#36c}.q{color:#00c}.ts td{padding:0}.ts{border-collapse:collapse}em{font-weight:bold;font-style:normal}.lst{height:25px;width:496px}.gsfi,.lst{font:18px arial,sans-serif}.gsfs{font:17px arial,sans-serif}.ds{display:inline-box;display:inline-block;margin:3px 0 4px;margin-left:4px}input{font-family:inherit}a.gb1,a.gb2,a.gb3,a.gb4{color:#11c !important}body{background:#fff;color:black}a{color:#11c;text-decoration:none}a:hover,a:active{text-decoration:underline}.fl a{color:#36c}a:visited{color:#551a8b}a.gb1,a.gb4{text-decoration:underline}a.gb3:hover{text-decoration:none}#ghead a.gb2:hover{color:#fff !important}.sblc{padding-top:5px}.sblc a{display:block;margin:2px 0;margin-left:13px;font-size:11px}.lsbb{background:#eee;border:solid 1px;border-color:#ccc #999 #999 #ccc;height:30px}.lsbb{display:block}.ftl,#fll a{display:inline-block;margin:0 12px}.lsb{background:url(/images/srpr/nav_logo80.png) 0 -258px repeat-x;border:none;color:#000;cursor:pointer;height:30px;margin:0;outline:0;font:15px arial,sans-serif;vertical-align:top}.lsb:active{background:#ccc}.lst:focus{outline:none}</style><script></script><link rel="stylesheet" type="text/css" media="all" href="/searchahead-js/css/searchaheadbox.css"/>\r\n<!--[if IE 6]>\r\n<link rel="stylesheet" type="text/css" media="all" href="/searchahead-js/css/searchaheadbox_ie6.css"/>\r\n<![endif]-->\r\n<!--[if IE 7]>\r\n<link rel="stylesheet" type="text/css" media="all" href="/searchahead-js/css/searchaheadbox_ie7.css"/>\r\n<![endif]-->\r\n\r\n<script type="text/javascript" src="/searchahead-js/searchaheadAll.min.js"></script></head><body bgcolor="#fff"><script>(function(){var src=\'/images/nav_logo199.png\';var iesg=false;document.body.onload = function(){window.n && window.n();if (document.images){new Image().src=src;}\nif (!iesg){document.f&&document.f.q.focus();document.gbqf&&document.gbqf.q.focus();}\n}\n})();</script><div id="mngb"> <div id=gbar><nobr><b class=gb1>Search</b> <a class=gb1 href="http://www.google.com/imghp?hl=en&tab=wi">Images</a> <a class=gb1 href="http://maps.google.com/maps?hl=en&tab=wl">Maps</a> <a class=gb1 href="https://play.google.com/?hl=en&tab=w8">Play</a> <a class=gb1 href="http://www.youtube.com/?tab=w1">YouTube</a> <a class=gb1 href="http://news.google.com/nwshp?hl=en&tab=wn">News</a> <a class=gb1 href="https://mail.google.com/mail/?tab=wm">Gmail</a> <a class=gb1 href="https://drive.google.com/?tab=wo">Drive</a> <a class=gb1 style="text-decoration:none" href="http://www.google.com/intl/en/options/"><u>More</u> »</a></nobr></div><div id=guser width=100%><nobr><span id=gbn class=gbi></span><span id=gbf class=gbf></span><span id=gbe></span> <div><style>.pmoabs{background-color:#fff;border:1px solid #E5E5E5;color:#666;font-size:13px;padding-bottom:20px;position:absolute;right:2px;top:3px;z-index:986}#pmolnk{border-radius:2px;-moz-border-radius:2px;-webkit-border-radius:2px}.kd-button-submit{border:1px solid #3079ed;background-color:#4d90fe;background-image:-webkit-gradient(linear,left top,left bottom,from(#4d90fe),to(#4787ed));background-image:-webkit-linear-gradient(top,#4d90fe,#4787ed);background-image:-moz-linear-gradient(top,#4d90fe,#4787ed);background-image:-ms-linear-gradient(top,#4d90fe,#4787ed);background-image:-o-linear-gradient(top,#4d90fe,#4787ed);background-image:linear-gradient(top,#4d90fe,#4787ed);filter:progid:DXImageTransform.Microsoft.gradient(startColorStr=\'#4d90fe\',EndColorStr=\'#4787ed\')}.kd-button-submit:hover{border:1px solid #2f5bb7;background-color:#357ae8;background-image:-webkit-gradient(linear,left top,left bottom,from(#4d90fe),to(#357ae8));background-image:-webkit-linear-gradient(top,#4d90fe,#357ae8);background-image:-moz-linear-gradient(top,#4d90fe,#357ae8);background-image:-ms-linear-gradient(top,#4d90fe,#357ae8);background-image:-o-linear-gradient(top,#4d90fe,#357ae8);background-image:linear-gradient(top,#4d90fe,#357ae8);filter:progid:DXImageTransform.Microsoft.gradient(startColorStr=\'#4d90fe\',EndColorStr=\'#357ae8\')}.kd-button-submit:active{-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.3);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.3);box-shadow:inset 0 1px 2px rgba(0,0,0,0.3)}#pmolnk a{color:#fff;display:inline-block;font-weight:bold;padding:5px 20px;text-decoration:none;white-space:nowrap}.xbtn{color:#999;cursor:pointer;font-size:23px;line-height:5px;padding-top:5px}.padi{padding:0 8px 0 10px}.padt{padding:5px 20px 0 0;color:#444}.pads{text-align:left;max-width:200px}</style> <div class="pmoabs" id="pmocntr2" style="behavior:url(#default#userdata);display:none"> <table border="0"> <tr> <td colspan="2"> <div class="xbtn" onclick="google.promos&&google.promos.toast&& google.promos.toast.cpc()" style="float:right">×</div> </td> </tr> <tr> <td class="padi" rowspan="2"> <img src="/images/icons/product/chrome-48.png"> </td> <td class="pads">A faster way to browse the web</td> </tr> <tr> <td class="padt"> <div class="kd-button-submit" id="pmolnk"> <a href="/chrome/index.html?hl=en&brand=CHNG&utm_source=en-hpp&utm_medium=hpp&utm_campaign=en" onclick="google.promos&&google.promos.toast&& google.promos.toast.cl()">Install Google Chrome </div> </td> </tr> </table> </div> <script type="text/javascript">(function(){var a={s:{}};a.s.va=50;a.s.ra=10;a.s.aa="body";a.s.Ra=!0;a.s.Ua=function(b,c){var d=a.s.Ha();a.s.Ja(d,b,c);a.s.Va(d);a.s.Ra&&a.s.Sa(d)};a.s.Va=function(b){(b=a.s.ca(b))&&0<b.forms.length&&b.forms[0].submit()};a.s.Ha=function(){var b=document.createElement("iframe");b.height=0;b.width=0;b.style.overflow="hidden";b.style.top=b.style.left="-100px";b.style.position="absolute";document.body.appendChild(b);return b};a.s.ca=function(b){return b.contentDocument||b.contentWindow.document};a.s.Ja=function(b,c,d){b=a.s.ca(b);b.open();d=["<",a.s.aa,\'><form method=POST action="\',d,\'">\'];for(var e in c)c.hasOwnProperty(e)&&d.push(\'<textarea name="\',e,\'">\',c[e],"</textarea>");d.push("</form></",a.s.aa,">");b.write(d.join(""));b.close()};a.s.ea=function(b,c){c>a.s.ra?google&&google.ml&&google.ml(Error("ogcdr"),!1,{cause:"timeout"}):b.contentWindow?a.s.Ta(b):window.setTimeout(function(){a.s.ea(b,c+1)},a.s.va)};a.s.Ta=function(b){document.body.removeChild(b)};a.s.Sa=function(b){a.s.Fa(b,"load",function(){a.s.ea(b,0)})};a.s.Fa=function(b,c,d){b.addEventListener?b.addEventListener(c,d,!1):b.attachEvent&&b.attachEvent("on"+c,d)};var m={Ya:0,G:1,H:2,M:5};a.o={};a.o.O={na:"i",L:"d",pa:"l"};a.o.C={P:"0",I:"1"};a.o.R={N:1,L:2,K:3};a.o.A={ha:"a",la:"g",F:"c",Ba:"u",Aa:"t",P:"p",ta:"pid",ja:"eid",Ca:"at"};a.o.oa=window.location.protocol+"//www.google.com/_/og/promos/";a.o.ka="g";a.o.Da="z";a.o.V=function(b,c,d,e){var f=null;switch(c){case m.G:f=window.gbar.up.gpd(b,d,!0);break;case m.M:f=window.gbar.up.gcc(e)}return null==f?0:parseInt(f,10)};a.o.Na=function(b,c,d){return c==m.G?null!=window.gbar.up.gpd(b,d,!0):!1};a.o.S=function(b,c,d,e,f,h,k,l){var g={};g[a.o.A.P]=b;g[a.o.A.la]=c;g[a.o.A.ha]=d;g[a.o.A.Ca]=e;g[a.o.A.ja]=f;g[a.o.A.ta]=1;k&&(g[a.o.A.F]=k);l&&(g[a.o.A.Ba]=l);if(h)g[a.o.A.Aa]=h;else return google.ml(Error("knu"),!1,{cause:"Token is not found"}),null;return g};a.o.Y=function(b,c,d){if(b){var e=c?a.o.ka:a.o.Da;c&&d&&(e+="?authuser="+d);a.s.Ua(b,a.o.oa+e)}};a.o.Ia=function(b,c,d,e,f,h,k){b=a.o.S(c,b,a.o.O.L,a.o.R.L,d,f,null,e);a.o.Y(b,h,k)};a.o.La=function(b,c,d,e,f,h,k){b=a.o.S(c,b,a.o.O.na,a.o.R.N,d,f,e,null);a.o.Y(b,h,k)};a.o.Qa=function(b,c,d,e,f,h,k,l,g,n){switch(c){case m.M:window.gbar.up.dpc(e,f);break;case m.G:window.gbar.up.spd(b,d,1,!0);break;case m.H:g=g||!1,l=l||"",h=h||0,k=k||a.o.C.I,n=n||0,a.o.Ia(e,h,k,f,l,g,n)}};a.o.Oa=function(b,c,d,e,f){return c==m.G?0<d&&a.o.V(b,c,e,f)>=d:!1};a.o.Ka=function(b,c,d,e,f,h,k,l,g,n){switch(c){case m.M:window.gbar.up.iic(e,f);break;case m.G:c=a.o.V(b,c,d,e)+1;window.gbar.up.spd(b,d,c.toString(),!0);break;case m.H:g=g||!1,l=l||"",h=h||0,k=k||a.o.C.P,n=n||0,a.o.La(e,h,k,1,l,g,n)}};a.o.Pa=function(b,c,d,e,f,h){b=a.o.S(c,b,a.o.O.pa,a.o.R.K,d,e,null,null);a.o.Y(b,f,h)};var p={Wa:"a",Za:"l",Xa:"c",ia:"d",K:"h",N:"i",jb:"n",I:"x",gb:"ma",hb:"mc",ib:"mi",$a:"pa",ab:"pc",cb:"pi",fb:"pn",eb:"px",bb:"pd",kb:"gpa",mb:"gpi",nb:"gpn",ob:"gpx",lb:"gpd"};a.i={};a.i.w={qa:"hplogo",za:"pmocntr2"};a.i.C={ya:"0",I:"1",ga:"2"};a.i.v=document.getElementById(a.i.w.za);a.i.ma=16;a.i.wa=2;a.i.xa=20;google.promos=google.promos||{};google.promos.toast=google.promos.toast||{};a.i.J=function(b){a.i.v&&(a.i.v.style.display=b?"":"none",a.i.v.parentNode&&(a.i.v.parentNode.style.position=b?"relative":""))};a.i.fa=function(b){try{if(a.i.v&&b&&b.es&&b.es.m){var c=window.gbar.rtl(document.body)?"left":"right";a.i.v.style[c]=b.es.m-a.i.ma+a.i.wa+"px";a.i.v.style.top=a.i.xa+"px"}}catch(d){google.ml(d,!1,{cause:a.i.B+"_PT"})}};google.promos.toast.cl=function(){try{a.i.T==m.H&&a.o.Pa(a.i.W,a.i.D,a.i.C.ga,a.i.$,a.i.X,a.i.Z),window.gbar.up.sl(a.i.D,a.i.B,p.K,a.i.U(),1)}catch(b){google.ml(b,!1,{cause:a.i.B+"_CL"})}};google.promos.toast.cpc=function(){try{a.i.v&&(a.i.J(!1),a.o.Qa(a.i.v,a.i.T,a.i.w.ba,a.i.W,a.i.Ga,a.i.D,a.i.C.I,a.i.$,a.i.X,a.i.Z),window.gbar.up.sl(a.i.D,a.i.B,p.ia,a.i.U(),1))}catch(b){google.ml(b,!1,{cause:a.i.B+"_CPC"})}};a.i.da=function(){try{if(a.i.v){var b=276,c=document.getElementById(a.i.w.qa);c&&(b=Math.max(b,c.offsetWidth));var d=parseInt(a.i.v.style.right,10)||0;a.i.v.style.visibility=2*(a.i.v.offsetWidth+d)+b>document.body.clientWidth?"hidden":""}}catch(e){google.ml(e,!1,{cause:a.i.B+"_HOSW"})}};a.i.Ea=function(){var b=["gpd","spd","aeh","sl"];if(!window.gbar||!window.gbar.up)return!1;for(var c=0,d;d=b[c];c++)if(!(d in window.gbar.up))return!1;return!0};a.i.Ma=function(){return a.i.v.currentStyle&&"absolute"!=a.i.v.currentStyle.position};google.promos.toast.init=function(b,c,d,e,f,h,k,l,g,n,q,r){try{if(!a.i.Ea())google.ml(Error("apa"),!1,{cause:a.i.B+"_INIT"});else if(a.i.v)if(e==m.H&&!l==!g)google.ml(Error("tku"),!1,{cause:"zwieback: "+g+", gaia: "+l}),a.i.J(!1);else if(a.i.w.F="toast_count_"+c+(q?"_"+q:""),a.i.w.ba="toast_dp_"+c+(r?"_"+r:""),a.i.B=d,a.i.D=b,a.i.T=e,a.i.W=c,a.i.Ga=f,a.i.$=l?l:g,a.i.X=!!l,a.i.Z=k,a.o.Na(a.i.v,e,a.i.w.ba,c)||a.o.Oa(a.i.v,e,h,a.i.w.F,c)||a.i.Ma())a.i.J(!1);else{a.o.Ka(a.i.v,e,a.i.w.F,c,f,a.i.D,a.i.C.ya,a.i.$,a.i.X,a.i.Z);if(!n){try{window.gbar.up.aeh(window,"resize",a.i.da)}catch(u){}window.lol=a.i.da;window.gbar.elr&&a.i.fa(window.gbar.elr());window.gbar.elc&&window.gbar.elc(a.i.fa);a.i.J(!0)}window.gbar.up.sl(a.i.D,a.i.B,p.N,a.i.U())}}catch(t){google.ml(t,!1,{cause:a.i.B+"_INIT"})}};a.i.U=function(){var b=a.o.V(a.i.v,a.i.T,a.i.w.F,a.i.W);return"ic="+b};})();</script> <script type="text/javascript">(function(){var sourceWebappPromoID=144002;var sourceWebappGroupID=5;var payloadType=5;var cookieMaxAgeSec=2592000;var dismissalType=5;var impressionCap=25;var gaiaXsrfToken=\'\';var zwbkXsrfToken=\'\';var kansasDismissalEnabled=false;var sessionIndex=0;var invisible=false;window.gbar&&gbar.up&&gbar.up.r&&gbar.up.r(payloadType,function(show){if (show){google.promos.toast.init(sourceWebappPromoID,sourceWebappGroupID,payloadType,dismissalType,cookieMaxAgeSec,impressionCap,sessionIndex,gaiaXsrfToken,zwbkXsrfToken,invisible,\'0612\');}\n});})();</script> </div> </span><br clear="all" id="lgpd"><div id="lga"><img alt="Eiji Tsuburaya’s 114th Birthday" border="0" height="200" src="/logos/doodles/2015/eiji-tsuburayas-114th-birthday-4809204506296320.2-hp.jpg" title="Eiji Tsuburaya’s 114th Birthday" width="356" id="hplogo" onload="window.lol&&lol()"><br><br></div><form action="/search" name="f"><table cellpadding="0" cellspacing="0"><tr valign="top"><td width="25%"> </td><td align="center" nowrap=""><input name="ie" value="ISO-8859-1" type="hidden"><input value="en" name="hl" type="hidden"><input name="source" type="hidden" value="hp"><input name="biw" type="hidden"><input name="bih" type="hidden"><div class="ds" style="height:32px;margin:4px 0"><input style="color:#000;margin:0;padding:5px 8px 0 6px;vertical-align:top" autocomplete="off" class="lst" value="" title="Google Search" maxlength="2048" name="q" size="57"></div><br style="line-height:0"><span class="ds"><span class="lsbb"><input class="lsb" value="Google Search" name="btnG" type="submit"></span></span><span class="ds"><span class="lsbb"><input class="lsb" value="I\'m Feeling Lucky" name="btnI" onclick="if(this.form.q.value)this.checked=1; else top.location=\'/doodles/\'" type="submit"></span></span></td><td class="fl sblc" align="left" nowrap="" width="25%">Advanced searchLanguage tools</td></tr></table><input id="gbv" name="gbv" type="hidden" value="1"></form><div id="gac_scont"></div><div style="font-size:83%;min-height:3.5em"><br></div><span id="footer"><div style="font-size:10pt"><div style="margin:19px auto;text-align:center" id="fll">Advertising ProgramsBusiness Solutions+GoogleAbout Google</div></div><p style="color:#767676;font-size:8pt">© 2015 - Privacy - Terms</p></span></center><script>(function(){window.google.cdo={height:0,width:0};(function(){var a=window.innerWidth,b=window.innerHeight;if(!a||!b)var c=window.document,d="CSS1Compat"==c.compatMode?c.documentElement:c.body,a=d.clientWidth,b=d.clientHeight;a&&b&&(a!=google.cdo.width||b!=google.cdo.height)&&google.log("","","/client_204?&atyp=i&biw="+a+"&bih="+b+"&ei="+google.kEI);})();})();</script><div id="xjsd"></div><div id="xjsi" data-jiis="bp"><script>(function(){function c(b){window.setTimeout(function(){var a=document.createElement("script");a.src=b;document.getElementById("xjsd").appendChild(a)},0)}google.dljp=function(b,a){google.xjsu=b;c(a)};google.dlj=c;})();(function(){window.google.xjsrm=[];})();if(google.y)google.y.first=[];if(!google.xjs){window._=window._||{};window._._DumpException=function(e){throw e};if(google.timers&&google.timers.load.t){google.timers.load.t.xjsls=new Date().getTime();}google.dljp(\'/xjs/_/js/k\\x3dxjs.hp.en_US.n4Isykx1pOI.O/m\\x3dsb_he,d/rt\\x3dj/d\\x3d1/t\\x3dzcms/rs\\x3dACT90oGruBlRL4zSGyBzh4XK8InqnND9gA\',\'/xjs/_/js/k\\x3dxjs.hp.en_US.n4Isykx1pOI.O/m\\x3dsb_he,d/rt\\x3dj/d\\x3d1/t\\x3dzcms/rs\\x3dACT90oGruBlRL4zSGyBzh4XK8InqnND9gA\');google.xjs=1;}google.pmc={"sb_he":{"agen":true,"cgen":true,"client":"heirloom-hp","dh":true,"ds":"","fl":true,"host":"google.com","jam":0,"jsonp":true,"msgs":{"cibl":"Clear Search","dym":"Did you mean:","lcky":"I\\u0026#39;m Feeling Lucky","lml":"Learn more","oskt":"Input tools","psrc":"This search was removed from your \\u003Ca href=\\"/history\\"\\u003EWeb History\\u003C/a\\u003E","psrl":"Remove","sbit":"Search by image","srch":"Google Search"},"ovr":{},"pq":"","refoq":true,"refpd":true,"rfs":[],"scd":10,"sce":5,"stok":"zjtqyOq-JYCOTkhf-6zvDRQ7OsI"},"d":{}};google.y.first.push(function(){if(google.med){google.med(\'init\');google.initHistory();google.med(\'history\');}});if(google.j&&google.j.en&&google.j.xi){window.setTimeout(google.j.xi,0);}\n</script></div></body></html>'
>>>
Sorry for the content is too long :) But it's really pure string.
I assume you are trying to open a url using selenium by creating a webdriver
profile = webdriver.FirefoxProfile()
browser = webdriver.Firefox(firefox_profile=profile)
browser.get(url)
In that case if you just say "browser.page_source" it should get the page source. But make sure that you wait until the page is loaded before reading the pagesource.
To test the behavior in your webpage. I suggest you use BDD Javascript Framework like CasperJs/PhantomJS