70,940 views / 2010.02.06 / 5:05 下午
做过好多抓取别家网站内容的产品,习惯了使用方便快捷的file_get_contents函数,但是总是会遇到获取失败的问题,尽管按照手册中的例子设置了超时,可多数时候不会奏效:
$config[‘context’] = stream_context_create(array(‘http’ => array(‘method’ => “GET”,
‘timeout’ => 5//这个超时时间不稳定,经常不奏效
)
));
这时候,看一下服务器的连接池,会发现一堆类似的错误,让你头疼万分:
file_get_contents(http://***): failed to open stream…
不得已,安装了curl库,写了一个函数替换:
function curl_file_get_contents($durl){
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $durl);
curl_setopt($ch, CURLOPT_TIMEOUT, 5);
curl_setopt($ch, CURLOPT_USERAGENT, _USERAGENT_);
curl_setopt($ch, CURLOPT_REFERER,_REFERER_);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$r = curl_exec($ch);
curl_close($ch);
return $r;
} |
function curl_file_get_contents($durl){
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $durl);
curl_setopt($ch, CURLOPT_TIMEOUT, 5);
curl_setopt($ch, CURLOPT_USERAGENT, _USERAGENT_);
curl_setopt($ch, CURLOPT_REFERER,_REFERER_);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$r = curl_exec($ch);
curl_close($ch);
return $r;
}
如此,除了真正的网络问题外,没再出现任何问题。
这是别人做过的关于curl和file_get_contents的测试:
file_get_contents抓取google.com需用秒数:
2.31319094
2.30374217
2.21512604
3.30553889
2.30124092
curl使用的时间:
0.68719101
0.64675593
0.64326
0.81983113
0.63956594
差距很大吧?呵呵,从我使用的经验来说,这两个工具不只是速度有差异,稳定性也相差很大。建议对网络数据抓取稳定性要求比较高的朋友使用上面的curl_file_get_contents函数,不但稳定速度快,还能假冒浏览器欺骗目标地址哦!
14,296 views / 2010.01.18 / 4:04 下午
我们在抓取网站内容的时候,经常遇到稀奇古怪的防盗链,比如上次碰到一个站的图片地址是假的,访问后要301跳转一次才到真正的图片路径,这个真实的路径又做了防盗措施,判断referer是不是上个假的图片地址。用curl试了几次,终于整出一个函数,效果不错。
$curl_loops = 0;//避免死了循环必备
$curl_max_loops = 3;
function curl_get_file_contents($url, $referer='') {
global $curl_loops, $curl_max_loops;
$useragent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
if ($curl_loops++ >= $curl_max_loops) {
$curl_loops = 0;
return false;
}
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);?curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_USERAGENT, $useragent);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_REFERER, $referer);
$data = curl_exec($ch);
$ret = $data;
list($header, $data) = explode("\r\n\r\n", $data, 2);
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$last_url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
curl_close($ch);
if ($http_code == 301 || $http_code == 302) {
$matches = array();
preg_match('/Location:(.*?)\n/', $header, $matches);
$url = @parse_url(trim(array_pop($matches)));
if (!$url) {
?$curl_loops = 0;
?return $data;
}
$new_url = $url['scheme'] . '://' . $url['host'] . $url['path']
. (isset($url['query']) ? '?' . $url['query'] : '');
$new_url = stripslashes($new_url);
return curl_get_file_contents($new_url, $last_url);
} else {
$curl_loops = 0;
list($header, $data) = explode("\r\n\r\n", $ret, 2);
return $data;
}
} |
$curl_loops = 0;//避免死了循环必备
$curl_max_loops = 3;
function curl_get_file_contents($url, $referer='') {
global $curl_loops, $curl_max_loops;
$useragent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
if ($curl_loops++ >= $curl_max_loops) {
$curl_loops = 0;
return false;
}
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);?curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_USERAGENT, $useragent);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_REFERER, $referer);
$data = curl_exec($ch);
$ret = $data;
list($header, $data) = explode("\r\n\r\n", $data, 2);
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$last_url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
curl_close($ch);
if ($http_code == 301 || $http_code == 302) {
$matches = array();
preg_match('/Location:(.*?)\n/', $header, $matches);
$url = @parse_url(trim(array_pop($matches)));
if (!$url) {
?$curl_loops = 0;
?return $data;
}
$new_url = $url['scheme'] . '://' . $url['host'] . $url['path']
. (isset($url['query']) ? '?' . $url['query'] : '');
$new_url = stripslashes($new_url);
return curl_get_file_contents($new_url, $last_url);
} else {
$curl_loops = 0;
list($header, $data) = explode("\r\n\r\n", $ret, 2);
return $data;
}
}
126,301 views / 2009.12.02 / 1:01 下午
好多人发来消息询问curl存取cookie文件的问题,杜工并不觉得这是个难点,因为只看手册就可以很容易把握。下面给个例子,看完后就全都明了了:
<?php
$cookie_jar_index = 'cookie.txt';
$url = "http://www.71j.cn/perl/login.pl";
$params = "username=dudu&password=****";
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_jar_index);
//curl_setopt($ch, CURLOPT_COOKIE, "fruit=apple; colour=red");
//上面代码是直接传递cookie信息,而非文件
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $params);
//curl_setopt($ch, CURLOPT_NOBODY, 1);//这个不能打开,否则无法生成cookie文件
ob_start();
curl_exec($ch);
curl_close($ch);
ob_clean();
$url = "http://www.71j.cn/perl/myfavorites.pl";
$ch2 = curl_init();
curl_setopt($ch2, CURLOPT_URL, $url);
curl_setopt($ch2, CURLOPT_COOKIEFILE, $cookie_jar_index);
ob_start();
curl_exec($ch2);
curl_close($ch2);
$rs = ob_get_contents(); //$rs就是返回的内容
ob_clean();
print_r($rs);
?> |
<?php
$cookie_jar_index = 'cookie.txt';
$url = "http://www.71j.cn/perl/login.pl";
$params = "username=dudu&password=****";
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_jar_index);
//curl_setopt($ch, CURLOPT_COOKIE, "fruit=apple; colour=red");
//上面代码是直接传递cookie信息,而非文件
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $params);
//curl_setopt($ch, CURLOPT_NOBODY, 1);//这个不能打开,否则无法生成cookie文件
ob_start();
curl_exec($ch);
curl_close($ch);
ob_clean();
$url = "http://www.71j.cn/perl/myfavorites.pl";
$ch2 = curl_init();
curl_setopt($ch2, CURLOPT_URL, $url);
curl_setopt($ch2, CURLOPT_COOKIEFILE, $cookie_jar_index);
ob_start();
curl_exec($ch2);
curl_close($ch2);
$rs = ob_get_contents(); //$rs就是返回的内容
ob_clean();
print_r($rs);
?>