php基于phpQuery的通用采集类
jerry PHP 2015年09月15日
收藏
<?php
/**
*通用列表采集类
*版本V1.3
*作者:JAE
*博客:http://blog.jaekj.com
*/
require_once '../phpQuery/phpQuery/phpQuery.php';
class QueryList{
private $pageURL;
private $regArr = array();
public $jsonArr = array();
private $regRange;
private $html;
/************************************************
* 参数: 页面地址 选择器数组 块选择器
* 【选择器数组】说明：格式array("名称"=>array("选择器","类型"),.......)
* 【类型】说明：值 "text" ,"html" ,"属性"
*【块选择器】：指 先按照规则 选出 几个大块 ，然后再分别再在块里面 进行相关的选择
*************************************************/
function QueryList($pageURL,$regArr=array(),$regRange='')
{
$this->pageURL = $pageURL;
//为了能获取https://
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL,$this->pageURL);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch,CURLOPT_RETURNTRANSFER,1);
$this->html = curl_exec($ch);
curl_close($ch);
if(!empty($regArr))
{
$this->regArr = $regArr;
$this->regRange = $regRange;
$this->getList();
}
}
function setQuery($regArr,$regRange='')
{
$this->jsonArr=array();
$this->regArr = $regArr;
$this->regRange = $regRange;
$this->getList();
}
private function getList()
{
$hobj = phpQuery::newDocumentHTML($this->html);
if(!empty($this->regRange))
{
$robj = pq($hobj)->find($this->regRange);
$i=0;
foreach($robj as $item)
{
while(list($key,$reg_value)=each($this->regArr))
{
$iobj = pq($item)->find($reg_value[0]);
switch($reg_value[1])
{
case 'text':
$this->jsonArr[$i][$key] = trim(pq($iobj)->text());
break;
case 'html':
$this->jsonArr[$i][$key] = trim(pq($iobj)->html());
break;
default:
$this->jsonArr[$i][$key] = pq($iobj)->attr($reg_value[1]);
break;
}
}
//重置数组指针
reset($this->regArr);
$i++;
}
}
else
{
while(list($key,$reg_value)=each($this->regArr))
{
$lobj = pq($hobj)->find($reg_value[0]);
$i=0;
foreach($lobj as $item)
{
switch($reg_value[1])
{
case 'text':
$this->jsonArr[$i++][$key] = trim(pq($item)->text());
break;
case 'html':
$this->jsonArr[$i++][$key] = trim(pq($item)->html());
break;
default:
$this->jsonArr[$i++][$key] = pq($item)->attr($reg_value[1]);
break;
}
}
}
}
}
function getJSON()
{
return json_encode($this->jsonArr);
}
}
<?php
require 'Query/QueryList.class.php';
//采集OSC的代码分享列表，标题 链接 作者
$url = "http://www.oschina.net/code/list";
$reg = array("title"=>array(".code_title a:eq(0)","text"),"url"=>array(".code_title a:eq(0)","href"),"author"=>array("img","title"));
$rang = ".code_list li";
$hj = new QueryList($url,$reg,$rang);
$arr = $hj->jsonArr;
print_r($arr);
//如果还想采当前页面右边的 TOP40活跃贡献者 图像，得到JSON数据,可以这样写
$reg = array("portrait"=>array(".hot_top img","src"));
$hj->setQuery($reg);
$json = $hj->getJSON();
echo $json . "<hr/>";
//采OSC内容页内容
$url = "http://www.oschina.net/code/snippet_186288_23816";
$reg = array("title"=>array(".QTitle h1","text"),"con"=>array(".Content","html"));
$hj = new QueryList($url,$reg);
$arr = $hj->jsonArr;
print_r($arr);
//就举这么多例子吧，是不是用来做采集很方便
<?php
/**
*自己写的百度和谷歌搜索API
*版本V2.0
*作者:JAE
*博客:http://blog.jaekj.com
**/
require_once 'QueryList_class.php';
class Searcher
{
private $searcher;
private $key;
private $num;
private $page;
private $regArr ;
private $regRange ;
private $regZnum;
public $jsonArr;
//参数 搜索引擎 搜索关键字 返回的结果条数 第几页
function Searcher($searcher,$key,$num,$page)
{
if($searcher=='baidu')
{
$this->regArr = array("title"=>array("h3.t a,#ting_singlesong_box a","text"),"tCon"=>array("div.c-abstract,font:slice(0,2),div#weibo,table tr:eq(0),div.c-abstract-size p:eq(0),div.vd_sitcom_new_tinfo","text"),"url"=>array("h3.t a,#ting_singlesong_box a","href"));
$this->regRange = 'table.result,table.result-op';
$this->regZnum=array("zNum"=>array("span.nums","text"));
}
else if($searcher=='google')
{
$this->regArr = array("title"=>array("h3.r a","text"),"tCon"=>array("span.st","text"),"url"=>array("h3.r a","href"));
$this->regRange = 'li.g';
$this->regZnum=array("zNum"=>array("div#resultStats","text"));
}
$this->searcher = $searcher;
$this->key = $key;
$this->num = $num;
$this->page = $page-1;
$this->getList();
}
private function getList()
{
$s = urlencode($this->key);
$num = $this->num;
$start = $this->num*$this->page;
if($this->searcher=='baidu')
{
$url = "http://www.baidu.com/s?pn=$start&rn=$num&wd=$s";
$reg_znum='/[\d,]+/';
}
else if($this->searcher=='google')
{
$url="https://www.google.com.hk/search?filter=0&lr=&newwindow=1&safe=images&hl=en&as_qdr=all&num=$num&start=$start&q=$s";
$reg_znum='/([\d,]+) result(s)?/';
}
$searcherObj = new QueryList($url,$this->regArr,$this->regRange);
for($i=0;$i<count($searcherObj->jsonArr);$i++)
{
if($this->searcher=='baidu')
{
$searcherObj->jsonArr[$i]['url'] = $this->getBaiduRealURL($searcherObj->jsonArr[$i]['url']);
}
else if($this->searcher=='google')
{
$searcherObj->jsonArr[$i]['url'] = $this->getGoogleRealURL($searcherObj->jsonArr[$i]['url']);
}
}
$this->jsonArr = $searcherObj->jsonArr ;
//获取总共结果条数
$searcherObj->setQuery($this->regZnum);
$zNum = $searcherObj->jsonArr[0]['zNum'];
preg_match($reg_znum,$zNum,$arr)?$zNum=$arr[0]:$zNum=0;
$zNum = (int)str_replace(',','',$zNum);
//计算总页数
$zPage = ceil($zNum/$this->num);
$this->jsonArr=array('num'=>$this->num,'page'=>((int)$this->page+1),'zNum'=>$zNum,'zPage'=>$zPage,"s"=>"$this->key",'other'=>array('author'=>'JAE','QQ'=>'734708094','blog'=>'http://blog.jaekj.com'),'data'=>$this->jsonArr);
}
function getJSON()
{
return json_encode($this->jsonArr);
}
private function getBaiduRealURL($url)
{
//得到百度跳转的真正地址
$header = get_headers($url,1);
if (strpos($header[0],'301') || strpos($header[0],'302'))
{
if(is_array($header['Location']))
{
//return $header['Location'][count($header['Location'])-1];
return $header['Location'][0];
}
else
{
return $header['Location'];
}
}
else
{
return $url;
}
}
private function getGoogleRealURL($url)
{
$reg_url = '/q=(.+)&/U';
return preg_match($reg_url,$url,$arr)?urldecode($arr[1]):$url;
}
}
// $hj = new Searcher('google','oschina',20,2);
// print_r( $hj->jsonArr);//因为执行时间较长，所以设置运行时间不被限制---编码utf-8 ini_set('max_execution_time', '0');
//ini_set('date.timezone','Asia/Taipei');
require_once dirname(__FILE__).'./phpQuery.php';
//比价网---採集
$http = 'http://www.beargoo.com.cn/';
$html = file_get_contents($http);
phpquery::newDocumentHTML($html,'utf-8');
//产品目录-----大类
$categorys = pq('div.menu li>a.drop');
foreach($categorys as $category){
$category = pq($category);
//大类名称---需数据库存储
$category_name = $category->text();
//大类---->子类产品---需数据库存储
$anchors = $category->next()->find('ul>li>a');
foreach($anchors as $anchor){
$anchor = pq($anchor);
//需数据库存储
$url = $anchor->attr('href');
// $type_url = parse_url($url);
// //产品类型
// $type = $type_url['query'];
//需数据库存储
$title = $anchor->text();
//在此处判断数据库中是否存在重复的记录，如果不存在，执行以下操作，存在就continue
$html = file_get_contents($url);
phpquery::newDocumentHTML($html,'utf-8');
//拿到全部的产品，模拟点击了全部按钮
$products_type = pq('dl.arrange')->find('dd:last');
preg_match_all("/\'(.*)\',\'(.*)\'/si",$products_type->attr('onclick'),$infos);
$param = $infos[1][0];
$var = $infos[2][0];
$all_url = $url.'&'.$param.'='.$var;
//重新发送请求
$html = file_get_contents($all_url);
phpquery::newDocumentHTML($html,'utf-8');
//获得产品的页数
$page_total = pq('div.pagination li:not(:last):last')->text();
//循环页数，拿到所有产品
for($i=1;$i<=$page_total;$i++){
$page_url = $all_url.'&page='.$i;
$html = file_get_contents($page_url);
phpquery::newDocumentHTML($html,'utf-8');
$product_lists = pq('div.list>div.productlist');
foreach($product_lists as $product_list){
$product_list = pq($product_list);
//产品url---需数据库存储
$product_url = $product_list->find('dt>a')->attr('href');
//产品名称---需数据库存储
$product_title = $product_list->find('dt>a')->text();
$html = file_get_contents($product_url);
phpquery::newDocumentHTML($html,'utf-8');
//比价网为您找到网销最低价---begin---需数据库存储
$lower_url = pq('dl.proparam')->find('dd>a:first')->attr('href');
$lower_price = pq('dl.proparam')->find('dd>a:first')->next()->text();
$lower_title = pq('dl.proparam')->find('dd>a:first')->text();
$detail_param_url = $http.pq('dl.proparam')->find('dd>a:last')->attr('href');
//end
//详细参数
$html = file_get_contents($detail_param_url);
phpquery::newDocumentHTML($html,'utf-8');
$params_cate = pq('div.paramb>div.paramhead>dt');
$params_paramk = pq('div.paramk');
foreach($params_paramk as $k=>$param_paramk){
$param_paramk = pq($param_paramk);
//参数类别---需数据库存储
$param_cate = $params_cate->eq($k);
$paramks = $param_paramk->find('dl');
foreach($paramks as $paramk){
$paramk = pq($paramk);
//参数名称---需数据库存储
$param_name = $paramk->find('dt')->text();
//参数值---需数据库存储
$param_value = $paramk->find('dd')->text();
//打印..此处做数据库保存工作..
exit($param_cate.$param_name.$param_value);
}
}
}
}
}
}推荐文章
