#node.js
#node.js
Вопрос:
Когда я делаю запрос к google.com обычно возвращается ответ, но для amazon.com это не так.
Посмотрите на результаты, которые я получаю:
Запрос Google
Ссылка на изображение запроса Google
Код:
const request = require('postman-request');
request({
method: 'get',
uri: 'https://www.google.com.br/',
headers: {
'User-Agent': 'postman-request',
'Accept': '*/*'
}
}, function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response amp;amp; response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.
});
Ответ консоли:
error: null
statusCode: 200
body: <!doctype html><html itemscope="" itemtype="http://schema.org/WebPage" lang="pt-BR"><head><meta content="text/html; charset=UTF-8" http-equiv="Content-Type"><meta content="/images/branding/googleg/1x/googleg_standard_color_128dp.png" itemprop="image"><title>Google</title><script nonce="s/WcsVhtsJfJxItYeVy0wg==">(function()æwindow.google=ækEI:'d6ydXPHzA72k5OUPg82_-As',kEXPI:'0,1353746,58,1958,2422,697,528,730,224,755,820,30,1227,1894,585,42,309,435,119,112,77,2333484,329563,1294,12383,4855,32692,15247,867,12163,16521,369,3314,1263,4242,2436,266,5107,575,835,284,2,1306,2431,1362,4323,4933,34,774,2256,4738,1151,2,1747,218,2595,3601,669,1050,1808,1397,81,7,2,489,620,29,1395,978,6722,1209,1288,2,4007,796,101,1119,38,920,754,119,1217,1362,352,1,133,47,1080,2693,43,293,1265,1503,2,631,2562,2,4,2,572,98,44,3744,560,354,125,2609,630,670,1119,439,656,338,254,2,526,810,1126,129,523,896,253,1314,705,756,91,7,392,29,400,39,334,2,203,310,105,455,53,27,571,10,168,8,87,22,187,7,836,71,152,78,365,817,174,47,465,45,6,257,147,48,578,10,4,50,304,216,581,505,287,9,2,464,10,2,64,6,597,380,25,283,172,3,42,5,478,13,569,102,90,67,420,54,99,200,388,129,54,187,132,213,387,41,25,98,39,240,11,57,147,32,185,2,22,5949943,2554,255,76,7,8797448,4,1572,549,333,444,1,2,80,1,900,575,17,304,1,8,1,2,2132,1,1,1,1,1,414,1,748,200,726,3,7,4,559,1,1320,157,1,123,3339038',authuser:0,kscs:'c9c918f0_d6ydXPHzA72k5OUPg82_-As',kGL:'BR'å;google.sn='webhp';google.kHL='pt-BR';å)();(function()ægoogle.lc=ÆÅ;google.li=0;google.getEI=function(a)æfor(var b;aamp;amp;(!a.getAttributeøø!(b=a.getAttribute("eid")));)a=a.parentNode;return bøøgoogle.kEIå;google.getLEI=function(a)æfor(var b=null;aamp;amp;(!a.getAttributeøø!(b=a.getAttribute("leid")));)a=a.parentNode;return bå;google.https=function()æreturn"https:"==window.location.protocolå;google.ml=function()æreturn nullå;google.time=function()æreturn(new Date).getTime()å;google.log=function(a,b,e,c,g)æif(a=google.logUrl(a,b,e,c,g))æb=new Image;var d=google.lc,f=google.li;dÆfÅ=b;b.onerror=b.onload=b.onabort=function()ædelete dÆfÅå;google.velamp;amp;google.vel.luamp;amp;google.vel.lu(a);b.src=a;google.li=f 1åå;google.logUrl=function(a,b,e,c,g)ævar d="",f=google.lsøø"";eøø-1!=b.search("amp;ei=")øø(d="amp;ei=" google.getEI(c),-1==b.search("amp;lei=")amp;amp;(c=google.getLEI(c))amp;amp;(d ="amp;lei=" c));c="";!eamp;amp;google.cshidamp;amp;-1==b.search("amp;cshid=")amp;amp;"slh"!=aamp;amp;(c="amp;cshid=" google.cshid);a=eøø"/" (gøø"gen_204") "?atyp=iamp;ct=" a "amp;cad=" b d f "amp;zx=" google.time() c;/Ühttp:/i.test(a)amp;amp;google.https()amp;amp;(google.ml(Error("a"),!1,æsrc:a,glmm:1å),a="");return aå;å).call(this);(function()ægoogle.y=æå;google.x=function(a,b)æif(a)var c=a.id;elseædo c=Math.random();while(google.yÆcÅ)ågoogle.yÆcÅ=Æa,bÅ;return!1å;google.lm=ÆÅ;google.plm=function(a)ægoogle.lm.push.apply(google.lm,a)å;google.lq=ÆÅ;google.load=function(a,b,c)ægoogle.lq.push(ÆÆaÅ,b,cÅ)å;google.loadAll=function(a,b)ægoogle.lq.push(Æa,bÅ)å;å).call(this);google.f=æå;var a=window.location,b=a.href.indexOf("#");if(0<=b)ævar c=a.href.substring(b 1);/(Üøamp;)q=/.test(c)amp;amp;-1==c.indexOf("#")amp;amp;a.replace("/search?" c.replace(/(Üøamp;)fp=ÆÜamp;Å*/g,"") "amp;cad=h")å;</script><style>#gbar,#guseræfont-size:13px;padding-top:1px !important;å#gbaræheight:22pxå#guseræpadding-bottom:7px !important;text-align:rightå.gbh,.gbdæborder-top:1px solid #c9d7f1;font-size:1pxå.gbhæheight:0;position:absolute;top:24px;width:100%åÄmedia allæ.gb1æheight:22px;margin-right:.5em;vertical-align:topå#gbaræfloat:leftååa.gb1,a.gb4ætext-decoration:underline !importantåa.gb1,a.gb4æcolor:#00c !importantå.gbi .gb4æcolor:#dd8e27 !importantå.gbf .gb4æcolor:#900 !importantå
</style><style>body,td,a,p,.hæfont-family:arial,sans-serifåbodyæmargin:0;overflow-y:scrollå#gogæpadding:3px 8px 0åtdæline-height:.8emå.gac_m tdæline-height:17pxåformæmargin-bottom:20pxå.hæcolor:#36cå.qæcolor:#00cå.ts tdæpadding:0å.tsæborder-collapse:collapseåemæfont-weight:bold;font-style:normalå.lstæheight:25px;width:496pxå.gsfi,.lstæfont:18px arial,sans-serifå.gsfsæfont:17px arial,sans-serifå.dsædisplay:inline-box;display:inline-block;margin:3px 0 4px;margin-left:4pxåinputæfont-family:inheritåa.gb1,a.gb2,a.gb3,a.gb4æcolor:#11c !importantåbodyæbackground:#fff;color:blackåaæcolor:#11c;text-decoration:noneåa:hover,a:activeætext-decoration:underlineå.fl aæcolor:#36cåa:visitedæcolor:#551a8båa.gb1,a.gb4ætext-decoration:underlineåa.gb3:hoverætext-decoration:noneå#ghead a.gb2:hoveræcolor:#fff !importantå.sblcæpadding-top:5pxå.sblc aædisplay:block;margin:2px 0;margin-left:13px;font-size:11pxå.lsbbæbackground:#eee;border:solid 1px;border-color:#ccc #999 #999 #ccc;height:30pxå.lsbbædisplay:blockå.ftl,#fll aædisplay:inline-block;margin:0 12pxå.lsbæbackground:url(/images/nav_logo229.png) 0 -261px repeat-x;border:none;color:#000;cursor:pointer;height:30px;margin:0;outline:0;font:15px arial,sans-serif;vertical-align:topå.lsb:activeæbackground:#cccå.lst:focusæoutline:noneå.tiahæwidth:458pxå</style><script nonce="s/WcsVhtsJfJxItYeVy0wg=="></script></head><body bgcolor="#fff"><script nonce="s/WcsVhtsJfJxItYeVy0wg==">(function()ævar src='/images/nav_logo229.png';var iesg=false;document.body.onload = function()æwindow.n amp;amp; window.n();if (document.images)ænew Image().src=src;å
if (!iesg)ædocument.famp;amp;document.f.q.focus();document.gbqfamp;amp;document.gbqf.q.focus();å
å
å)();</script><div id="mngb"> <div id=gbar><nobr><b class=gb1>Pesquisa</b> <a class=gb1 href="https://www.google.com.br/imghp?hl=pt-BRamp;tab=wi">Imagens</a> <a class=gb1 href="https://maps.google.com.br/maps?hl=pt-BRamp;tab=wl">Maps</a> <a class=gb1 href="https://play.google.com/?hl=pt-BRamp;tab=w8">Play</a> <a class=gb1 href="https://www.youtube.com/?gl=BRamp;tab=w1">YouTube</a> <a class=gb1 href="https://news.google.com.br/nwshp?hl=pt-BRamp;tab=wn">Not�cias</a> <a class=gb1 href="https://mail.google.com/mail/?tab=wm">Gmail</a> <a class=gb1 href="https://drive.google.com/?tab=wo">Drive</a> <a class=gb1 style="text-decoration:none" href="https://www.google.com.br/intl/pt-BR/about/products?tab=wh"><u>Mais</u> amp;raquo;</a></nobr></div><div id=guser width=100%><nobr><span id=gbn class=gbi></span><span id=gbf class=gbf></span><span id=gbe></span><a href="http://www.google.com.br/history/optout?hl=pt-BR" class=gb4>Hist�rico da Web</a> ø <a href="/preferences?hl=pt-BR" class=gb4>Configura��es</a> ø <a target=_top id=gb_70 href="https://accounts.google.com/ServiceLogin?hl=pt-BRamp;passive=trueamp;continue=https://www.google.com.br/" class=gb4>Fazer login</a></nobr></div><div class=gbh style=left:0></div><div class=gbh style=right:0></div> </div><center><br clear="all" id="lgpd"><div id="lga"><img alt="Google" height="92" src="/images/branding/googlelogo/1x/googlelogo_white_background_color_272x92dp.png" style="padding:28px 0 14px" width="272" id="hplogo" onload="window.lolamp;amp;lol()"><br><br></div><form action="/search" name="f"><table cellpadding="0" cellspacing="0"><tr valign="top"><td width="25%">amp;nbsp;</td><td align="center" nowrap=""><input name="ie" value="ISO-8859-1" type="hidden"><input value="pt-BR" name="hl" type="hidden"><input name="source" type="hidden" value="hp"><input name="biw" type="hidden"><input name="bih" type="hidden"><div class="ds" style="height:32px;margin:4px 0"><div style="position:relative;zoom:1"><input style="color:#000;margin:0;padding:5px 8px 0 6px;vertical-align:top;padding-right:38px" autocomplete="off" class="lst tiah" value="" title="Pesquisa Google" maxlength="2048" name="q" size="57"><img src="/textinputassistant/tia.png" style="position:absolute;cursor:pointer;right:5px;top:4px;z-index:300" data-script-url="/textinputassistant/11/pt-BR_tia.js" alt="" height="23" onclick="var s=document.createElement('script');s.src=this.getAttribute('data-script-url');(document.getElementById('xjsc')øødocument.body).appendChild(s);" width="27"></div></div><br style="line-height:0"><span class="ds"><span class="lsbb"><input class="lsb" value="Pesquisa Google" name="btnG" type="submit"></span></span><span class="ds"><span class="lsbb"><input class="lsb" value="Estou com sorte" name="btnI" onclick="if(this.form.q.value)this.checked=1; else top.location='/doodles/'" type="submit"></span></span></td><td class="fl sblc" align="left" nowrap="" width="25%"><a href="/advanced_search?hl=pt-BRamp;amp;authuser=0">Pesquisa avan�ada</a><a href="/language_tools?hl=pt-BRamp;amp;authuser=0">Ferramentas de idioma</a></td></tr></table><input id="gbv" name="gbv" type="hidden" value="1"><script nonce="s/WcsVhtsJfJxItYeVy0wg==">(function()ævar a,b="1";if(documentamp;amp;document.getElementById)if("undefined"!=typeof XMLHttpRequest)b="2";else if("undefined"!=typeof ActiveXObject)ævar c,d,e=Æ"MSXML2.XMLHTTP.6.0","MSXML2.XMLHTTP.3.0","MSXML2.XMLHTTP","Microsoft.XMLHTTP"Å;for(c=0;d=eÆc Å;)tryænew ActiveXObject(d),b="2"åcatch(h)æååa=b;if("2"==aamp;amp;-1==location.search.indexOf("amp;gbv=2"))ævar f=google.gbvu,g=document.getElementById("gbv");gamp;amp;(g.value=a);famp;amp;window.setTimeout(function()ælocation.href=få,0)å;å).call(this);</script></form><div id="gac_scont"></div><div style="font-size:83%;min-height:3.5em"><br></div><span id="footer"><div style="font-size:10pt"><div style="margin:19px auto;text-align:center" id="fll"><a href="/intl/pt-BR/ads/">Solu��es de publicidade</a><a href="/services/">Solu��es empresariais</a><a href="/intl/pt-BR/about.html">Sobre o Google</a><a href="https://www.google.com.br/setprefdomain?prefdom=USamp;amp;sig=K_uN4YO80CcIzoOtmOXUTtIim9uwg=" id="fehl">Google.com</a></div></div><p style="color:#767676;font-size:8pt">amp;copy; 2019 - <a href="/intl/pt-BR/policies/privacy/">Privacidade</a> - <a href="/intl/pt-BR/policies/terms/">Termos</a></p></span></center><script nonce="s/WcsVhtsJfJxItYeVy0wg==">(function()æwindow.google.cdo=æheight:0,width:0å;(function()ævar a=window.innerWidth,b=window.innerHeight;if(!aøø!b)ævar c=window.document,d="CSS1Compat"==c.compatMode?c.documentElement:c.body;a=d.clientWidth;b=d.clientHeightåaamp;amp;bamp;amp;(a!=google.cdo.widthøøb!=google.cdo.height)amp;amp;google.log("","","/client_204?amp;atyp=iamp;biw=" a "amp;bih=" b "amp;ei=" google.kEI);å).call(this);å)();(function()ævar u='/xjs/_/js/kØx3dxjs.hp.en.uZI4hC4e72c.O/mØx3dsb_he,d/amØx3dBIOa/rtØx3dj/dØx3d1/rsØx3dACT90oFEUUM4gw3uNTwHrphS2J65-WJjAQ';setTimeout(function()ævar a=document.createElement("script");a.src=u;google.timersamp;amp;google.timers.loadamp;amp;google.tickamp;amp;google.tick("load","xjsls");document.body.appendChild(a)å,0);å)();(function()æwindow.google.xjsu='/xjs/_/js/kØx3dxjs.hp.en.uZI4hC4e72c.O/mØx3dsb_he,d/amØx3dBIOa/rtØx3dj/dØx3d1/rsØx3dACT90oFEUUM4gw3uNTwHrphS2J65-WJjAQ';å)();function _DumpException(e)æthrow e;å
(function()ævar pmc='æØx22Qnk92gØx22:æå,Øx22RWGcrAØx22:æå,Øx22U5B21gØx22:æå,Øx22YFCs/gØx22:æå,Øx22ZI/YVQØx22:æå,Øx22dØx22:æå,Øx22sb_heØx22:æØx22agenØx22:true,Øx22cgenØx22:true,Øx22clientØx22:Øx22heirloom-hpØx22,Øx22dhØx22:true,Øx22dhqtØx22:true,Øx22dsØx22:Øx22Øx22,Øx22ffqlØx22:Øx22pt-BRØx22,Øx22flØx22:true,Øx22hostØx22:Øx22google.com.brØx22,Øx22isbhØx22:28,Øx22jsonpØx22:true,Øx22msgsØx22:æØx22ciblØx22:Øx22Limpar pesquisaØx22,Øx22dymØx22:Øx22Voc� quis dizer:Øx22,Øx22lckyØx22:Øx22Estou com sorteØx22,Øx22lmlØx22:Øx22Saiba maisØx22,Øx22osktØx22:Øx22Ferramentas de inser��o de textoØx22,Øx22psrcØx22:Øx22Esta pesquisa foi removida do seuØØu003Ca hrefØx3dØØØx22/historyØØØx22ØØu003EHist�rico da webØØu003C/aØØu003EØx22,Øx22psrlØx22:Øx22RemoverØx22,Øx22sbitØx22:Øx22Pesquisa por imagemØx22,Øx22srchØx22:Øx22Pesquisa GoogleØx22å,Øx22ovrØx22:æå,Øx22pqØx22:Øx22Øx22,Øx22refpdØx22:true,Øx22rfsØx22:ÆÅ,Øx22sbplØx22:24,Øx22sbprØx22:24,Øx22scdØx22:10,Øx22sceØx22:5,Øx22stokØx22:Øx22eMIPRCTn3u0Hqqwsig_8_01itAwØx22,Øx22uhdeØx22:falseåå';google.pmc=JSON.parse(pmc);å)();</script> </body></html>
Запрос Amazon
Ссылка на изображение запроса Amazon
Код:
const request = require('postman-request');
request({
method: 'get',
uri: 'https://www.amazon.com.br/',
headers: {
'User-Agent': 'postman-request',
'Accept': '*/*'
}
}, function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response amp;amp; response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.
});
Ответ консоли:
...
�O���4e�5���9�X�Ç�öl�[e�
�<#������p �"�� rr��C�_��.".��_�|>�]���z�[�����rh��Q)7$_�� �e3v���rh�Q�r�'M|��:z��v��1�2
� S�����W�,�P-�@,�S�;v� ��L�g�i�yS՚
G���Ј>��dn��g����xd[eS�:?;>X����=���V|0ב��,AJP��V�R���amp;�ݝ3�-��X�w<.�5���C�m�Ö�>]`{���ߠ?E�T���'Q�3
�X��:����H���Y����3�wu�����/��/k��7��}�� �*ٮ��Y0SI�1U������ͦT���Xv����U����]p�
�KbdSH�
l��c�<@�J燂Q�<��D��ڣ�
�2��L��I'%g���W ��d�_���������D���k-���`|����bg
k�)�_��S�G-�Z��D�Fv���<�<� ��cH�~Z�s��.�ȋqG�vI�^#���t���������������ӣ�3�GG�g���2�������D�/�ʯp~�_���?����i���*7����Nϟ���O���u����/������oO��~�|���}��N�Wt��"amp;m��9yz՟���˳�by��a��g��^w���)ff��'���?���ޑ�Z� �@�z])�sAz��z�6
����0Q����xM ͥP�Pn>ͰJ
pJJN r
H
t �L�r4N6�-
w�2
��
�
u7̲�
L��t�r��-5
,�
��/L��
��r�/Hs�J��pI�Ls
J���v�����
�L
r��@B��
Я хотел бы понять, почему, я предполагаю, что это кодировка, но я не знаю, как преобразовать в кодировку, которая делает ответ читаемым.
Ответ №1:
Добавьте заголовок 'Accept-Encoding': 'identity' при запросе amazon. Смотрите ниже код редактирования.
const request = require('postman-request');
request({
method: 'get',
uri: 'https://www.amazon.com.br/',
headers: {
'User-Agent': 'postman-request',
'Accept-Encoding': 'identity'
}
}, function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response amp;amp; response.statusCode); // Print the response status code if a response was received
console.log('body:',body); // Print the HTML for the Google homepage.
});
Надеюсь, это поможет.
Ответ №2:
Если вы перейдете к amazon.com.br в вашем браузере и посмотрите на сетевой запрос, вы увидите, что ответ имеет заголовок: content-encoding: gzip . Это означает, что, как вы сказали, ответ кодируется с помощью gzip . поскольку postman-request фактически это форк request пакета, вы можете увидеть опции для этого пакета: есть gzip опция, которая:
…если true, добавьте заголовок Accept-Encoding, чтобы запросить сжатые кодировки содержимого с сервера (если они еще не присутствуют) и декодировать поддерживаемые кодировки содержимого в ответе.
Кстати, просто для развлечения, помните, что вы всегда можете сделать это вручную, используя native node.js модули, подобные этому:
const zlib = require('zlib');
const https = require('https');
const fs = require('fs');
const request = https.get({
host: 'www.amazon.com.br',
path: '/',
headers: {'Accept-Encoding': 'gzip'}
});
request.on('response', (response) => {
const output = fs.createWriteStream('index.html');
response.pipe(zlib.createGunzip()).pipe(output);
});