?? cURL? ?? ??? ?? ??? ?????. PHP ??? http://us2.php.net/manual/zh/function.curl-multi-select.php ?? ???? ??? ???, ?? ?? ??? ?? ???? ??? ? ?? ??? ????. ??? ? ?? ???? ???? muti_curl ????? 一個是運?? ??? ??, 這里是批weight檢查代理ip是否可用
- class request_setting {
- public $url = false;
- public $method = 'GET';
- public $ post_data = null;
- ?? $headers = null;
- ?? $options = null;
- ?? __construct($url, $method = "GET", $post_data = null, $headers = null, $options = null) {
- $this->url = $url;
- $this->method = $method;
- $this->post_data = $post_data;
- $this-> ;headers = $headers;
- $this->options = $options;
- }
- ?? ?? __destruct() {
- unset($this->url, $this->method , $this->post_data, $this->headers, $this->options);
- }
- }
- /**************************************************** * *****************************************
- ?? ?? ???
- ************************************************ **** ********************************************/
- class muti_curl {
- protected $thread_size = 100;
- protected $timeout = 30;
- private $callback;
- protected $options = array(
- CURLOPT_SSL_VERIFYPEER => false,//禁用后cURL將終止從服務(wù)端進(jìn)行驗證。使用 CURLOPT_CAINFO選項設(shè)置證書使用 CURLOPT_CAPATH選項設(shè)置證書目錄 如果CURLOPT_SSL_VERIFYPEER(默認(rèn)值為2)被啟用CURLOPT_SSL_VERIFYHOST需要被設(shè)置成TRUE否則設(shè)置為FALSE。 自cURL 7.10開始默認(rèn)為TRUE。從cURL 7.10開始默認(rèn)綁 ? ?? 。
- CURLOPT_RETURNTRANSFER => true, //CURLOPT_CONNECTTIMEOUT => 15,
- CURLOPT_TIMEOUT => 30,
- // CURLOPT_HTTP_VERSION=>CURL_HTTP_VERSION_1_0, //???? ???? 時候用這個去抓取數(shù)據(jù),更爽
- // CURLOPT_AUTOREFERER=>false,// :????,?? ?? ? ???: 信息.
- // CURLOPT_BINARYTRANSFER=>false, //?? CURLOPT_RETURNTRANSFER? 時候, 返回原生的(Raw)? ???? ????.
- // ESSION=>??,/ / 啟用時curl會僅僅傳遞一個session cookie,忽略其他的cookie,默認(rèn)狀況下cURL會將所有的cookie返回
- // CURLOPT_CRLF=>false,// 啟用時將Unix?換行符轉(zhuǎn)換成回車換行符.
- // CURLOPT_DNS_USE_GLOBAL_CACHE=>false, // 啟用時會啟用一個全局的DNS緩存,此項為線程??? ?????.
- // FAILONERROR=>false, //顯示HTTP狀態(tài)碼,默認(rèn)行為是忽略編號小于等于400的HTTP信息。
- // CURLOPT_FILETIME=>true, //啟用時會嘗試修改遠(yuǎn)程文檔中??? ?? ???????. CURLINFO_FILETIME? ???????.curl_getinfo().
- // CURLOPT_FOLLOWLOCATION=>false, // 啟用時會將服務(wù)器服務(wù)器返回的"??: "放在header中遞歸的返回給服務(wù)器,使用CURLOPT_MAXREDIRS可以遞歸返回? ?? ?????.
- // CURLOPT_FORBID_REUSE =>true, //??? ???? ????.
- // CURLOPT_FRESH_CONNECT=>true,//
- // CURLOPT_FTP_USE_EPRT=>false,// 啟用時當(dāng)FTP下載時,使用EPRT (或 LPRT)命令。設(shè)置為FALSE時禁用EPRT?LPRT,使用PORT命令? ?????.
- // CURLOPT_FTP_USE_EPSV=>false,// FTP傳輸過程中回復(fù)到PASV模式前首先嘗試EPSV命令。設(shè)置為FALSE時禁用EPSV命令。
- // CURLOPT_FTPAPPEND=>false, // ?? ??? 時追加寫入文件而不是覆蓋它.
- // CURLOPT_FTPASCII=>false,// CURLOPT_TRANSFERTEXT? ??.
- // CURLOPT_FTPLISTONLY=>false,// ?? ??? ?? ?? FTP目錄 ?? ?????.
- // CURLOPT_HEADER=>true,// ???? ?? ??? ??? ????.
- // CURLINFO_HEADER_OUT=>false, //求字符串。
- // CURLOPT_HTTPGET=>true,// GET, 因為GET是默認(rèn)是, 所以只被修改情況下使用.
- // ? => ?? ,// 啟用時會HTTP代理來傳輸.
- // CURLOPT_MUTE=>true,// 啟用時將cURL函數(shù)中所有修改過的參數(shù)恢復(fù)默認(rèn)值。
- // OPT_NETRC=>?? ,// ?? 連接建立以后,訪問~/.netrc文件獲取用戶name and 密碼信息連接遠(yuǎn)程站點。
- // CURLOPT_NOBODY=>true, ?? ??? ???????.
- / / CURLOPT_NOPROGRESS=>false,//???? ?? ????? ?????.
- // CURLOPT_NOSIGNAL=>false,//傳遞給php? ??信號。? ??? SAPI ????? ?? ?? ????? ??????. cURL 7.10? ???????.
- // CURLOPT_POST=>false,// ????? ?? ??? ????? ?? POST ??? ?????(application/x-www-form-urlencoded ??).
- // CURLOPT_PUT=>false,// ????? HTTP? ??? ?? ? ??? ?????. CURLOPT_INFILE ? CURLOPT_INFILESIZE? ??? ???? ???.
- // CURLOPT_TRANSFERTEXT=>false,// ????? FTP ??? ASCII ??? ?????. LDAP? ?? HTML? ?? ?? ??? ??? ?????. Windows ?????? ???? STDOUT? ???? ??? ???? ????.
- // CURLOPT_UNRESTRICTED_AUTH=>true,// ??? ??? ??? ???? CURLOPT_FOLLOWLOCATION? ???? ??? ??? ?? ??? ??? ?? ? ???? ??? ????? ?????.
- // CURLOPT_UPLOAD=>false,// ????? ?? ???? ?????.
- // CURLOPT_VERBOSE =>true,// ????? ?? ??? ???? STDERR ?? ??? CURLOPT_STDERR? ?????.
- );
- private $headers = array();
- private $requests = array();
- private $requestMap = array();
- /*********************
- ?? ?? ??
- ********************/
- ?? __construct($callback = null) {
- $this->callback = $callback;
- }
-
- /**************************************************** * ****************
- __get ??? ????
- *********************** * **********************************************/
- ?? ?? __get($ name) {
- return (isset($this->{$name})) ? $this->{$name} : null;
- }
-
- /**************************************************** * *********************
- __set ??? ????
- ****************** *** **********************************/
- ?? ?? __set($name, $value) {
- // 增加一個設(shè)置到headers
- if ($name == "options" || $name == "headers") {
- $this->{$name} = $value $this->{$name };
- } else {
- $this->{$name} = $value;
- }
- return true;
- }
- //增加一個請求
- ?? function add($request) {
- $this->requests[] = $request;
- return true;
- }
-
-
- ?? ?? ??($url, $method = "GET", $post_data = null, $headers = null, $options = null) {
- $this->requests[] = new request_setting($url, $method, $post_data, $headers, $options );
- true ??;
- }
-
- ?? ?? get($url, $headers = null, $options = null) {
- return $this->request($url, "GET", null, $headers, $options);
- }
-
- ?? ?? post($url, $post_data = null, $headers = null, $options = null) {
- ?? $this->request($url, "POST", $post_data, $headers, $options);
- }
-
- ??? ?? Single_curl() {
- $ch = cur_init(); //初始化
- $request = array_shift($this->requests);//把第一個單元移出并作為結(jié)果
- $options = $this->get_options($request);//獲得該單代設(shè)置
- ?_setopt_array($ch, $options);//批設(shè)置
- $output = ?_exec($ch);
- $curl_info = ?_getinfo($ch);
-
- if ($this->callback) {
- $callback = $this->callback;
- if (is_callable($this->callback)) {
- call_user_func($callback, $output, $curl_info, $request);
- }
- }
- else
- return $output;
- return true;
- }
-
-
- ??? ?? Rolling_curl($ thread_size = null) {
- if ($thread_size){
- $this->thread_size = $thread_size;
- }
-
- if (count($this->requests) < $this->thread_size){
- $this->thread_size = count($this->requests);
- }
- if ($this->thread_size < 2) {
- $errorinfo = '線程大小必須大于 1!!!!';
- throw new Exception($errorinfo);
- }
- $queue =curl_multi_init();
-
- //?線程里開始增加任務(wù)隊列
- for ($i = 0; $i < $this->thread_size; $i ) {
- $ch = ?_init();
- $options = $this->get_options($this->requests[$i]);
- ?_setopt_array($ch, $options) ;//獲得設(shè)置
- cur_multi_add_handle($queue, $ch);//添加進(jìn)去
- $key = (string) $ch;
- $this->requestMap[$key] = $i;
- }
- do {
- while (($statu_run_muti_exec = cur_multi_exec($queue, $active)) == CURLM_CALL_MULTI_PERFORM) ;
- if ($statu_run_muti_exec != CURLM_OK){ break; }
- // 發(fā)現(xiàn)完成的一個請求,進(jìn)行處理
- while ($done =curl_multi_info_read($queue)) {
- $curl_info =curl_getinfo($done['handle']);
- $ ?? = ?_??_getcontent($done['handle']);
- $callback = $this->callback;
- if (is_callable($callback)){
- $key = (???) $done[ 'handle'];
- $request = $this->requests[$this->requestMap[$key]];
- unset($this->requestMap[$key]);//這個銷毀變一個未處理的請求加入到一個已經(jīng)完成成隊列中
- if ($i < count($this->requests) && isset($this->requests[$i]) && $i < count($this->requests)) {
- $ch =curl_init();
- $options = $this->get_options($this->requests[$i]);
- curl_setopt_array($ch, $options);
- curl_multi_add_handle($queue, $ch);
- $key = (???) $ch;
- $this->requestMap[$key] = $i;
- $i ;
- }
- cur_multi_remove_handle($queue, $done['handle']);
- echo "done ";
- print_r($queue);
- print_r ($done);
- }
- // ? ??? ?? ?????. ?? ? ??? ???? ?? ?? ??? ????? ???.
- // ??? ??? ?? ?? ????? ??? ??? ??? ? ?? ??? ???? ???? ????. ??? ??? ???? ?? ?? ???? ?? $active=0? ??? ??? ???? ????.
- if ($active >0){
- cur_multi_select($queue, $this ->timeout );
- }
-
- } while ($active);
- curl_multi_close($queue);
- return true;
- }
- ?? ?? ??($ thread_size = null) {
- //thread_size? ??? ?????. ??? ??? ?? ?? ?? ??? ??? ?????
- if (count($this->requests) == 1) {
- return $this->single_curl( );
- } else {
- return $this->rolling_curl($thread_size);
- }
- }
- ??? ?? get_options($request) {
- $options = $this ->__get('options');
- if (ini_get('safe_mode') == 'Off' || !ini_get('safe_mode')) {
- // $options[CURLOPT_FOLLOWLOCATION] = 1;
- // $options[CURLOPT_MAXREDIRS] = 5;
- }
- $headers = $this->__get('headers');
-
- if ($request->options) {
- $options = $request->options $options;
- }
- $options[CURLOPT_URL] = $request->url;
- //The ??? ?? ?? ??? ?? ?????. ;
- }
- if ( $headers) {
- $options[CURLOPT_HEADER] = 0;
- $options[CURLOPT_HTTPHEADER] = $headers;
- }
- return $options;
- }
-
- ?? ?? __destruct() {
- unset($this->thread_size, $this->callback, $this->options, $this->headers, $this- >??);
- }
- }
- ?>
-
-
- ?? ??
-
header("content-type:text/html; charset=utf-8");
- require("muti_curl_class.php");set_time_limit(0);
- $sucesesnum=0;
- $good_proxy=array();
- function request_callback($response, $info, $request) {
- global $ ;
- // 下面的正規(guī)可以選擇性地顯示回傳的結(jié)果
- /* if (preg_match("~(.*?)~i", $response, $out )) {
- $title = $out[1];
- }*/
- // echo '
'.$response .' ';
- echo '
';
- //對回應(yīng)也就是 $response 進(jìn)行偵測判斷裡面是否有設(shè)定的字符,如果有判斷運用該代理成功
- if( $response !== false && substr_count($response, 'User -agent: Baiduspider') >=1 ) {
- // $result = true;
- echo "true
";
- // echo $request[options][10004];
- // print_r ($request->options);
- echo $request->options[CURLOPT_PROXY];
- $good_proxy[]=$request->options[CURLOPT_PROXY];
- }
- }
- echo '
the-->'. $sucesesnum.'// print_r ($request);
- //echo $request-> url;
- $sucesesnum ;
- echo "
";
- }
- $params = array_merge($_GET, $_POST); //此處取得傳遞過來的代理ip的位址
- $result = $proxy_ip = trim($params['ip']);
- $timeout=intval(trim($params['timeout']));
- if($timeoutif($timeout>300){$timeout=300;}
- $thread_size=intval(trim($params['thread_size']));
- if($thread_sizeize if($thread_size>300){$thread_size =300;}
-
- if($proxy_ip == '') {
- echo '請輸入IP!!';
- return;
- }
- $replace_arr1 = array('?', 'qq代理:', 'dn28.com', 'qqip', 'qq代理', 'qqqipip ', '代理ip:', 'ip:', '代理ip','"',"'",'\','/',' ');
- $result = str_replace($replace_arr1, array (''), $result);
- $result = str_replace(",", "n", $result);
- $resArr = explode("n", $result);
- foreach( $resArr as $k => $v) {
- $posProxy = getPos($v, '@');
- if($posProxy===false){
- if (!empty($v )){$proxyip_and_port = $v; }
- }else{
- $proxyip_and_port = substr($v, 0, $posProxy);
- }
- $newRes[] =trim($proxyip_and_port) ;
- }
- print_r($newRes);
- //die();
- $option_setting = array(
- CURLOPT_SSL_VERIFYPEER => 0,
- CURLOPT_SSL_VERIFYPEER => 0, 5,
- CURLOPT_TIMEOUT => 30,
- CURLOPT_HEADER=>false,
- CURLOPT_PROXY=>'',///這個地方設(shè)定代理的位置
- );$btime=time();
- $rc = new muti_curl("request_callback");
- $rc->timeout = $timeout ;
- $rc->thread_size = $thread_size;
- foreach ($newRes as $v) {
- $option_setting[CURLOPT_PROXY]=$v;
- $request = new request_ $setting($, $setting( method = "GET", $post_data = null,$header= null, $option_setting);
- $rc->add($request);
- }
- $rc->execute();
- $etime=time();
- $usedtime=$etime-$btime;
- echo 'all'. $sucesesnum.'use'. $usedtime;
- echo '
';
- $good_proxy= array_unique($good_proxy);
- $str='';
- foreach ($good_proxy as $v){
- $str.="'".trim($v)."' ,";
- }
- $str= str_replace ( ' ' , '' ,$str );
- $str = preg_replace('/s /', ' ', $str);
- echo $str.'
';
- var_export ($good_proxy);
- //var_dump ($good_proxy);
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- //*************************************** ************************************************** *********************
- //************************ *******只用了一個函數(shù)
-
- function parseProxyInfo ( $proxyStr ) {
- //$proxyStr = '202.115.207.25:80@HTTP;四川省成都市四川師範(fàn)大學(xué)' ;
- $posIp = getPos($proxyStr, ':');
- $ip = substr($proxyStr, 0, $posIp);
- $posPort = getPos($proxyStr, '@');
- $port = substr($proxyStr, $posIp 1, $posPort-$posIp-1);
- $posType = getPos($proxyStr, ';');
- $type = substr($proxyStr , $posPort 1, $posType-$posPort-1);
- $location = substr(strstr($proxyStr, ';'), 1);
- return array(
- 'ip' => $ ip,
- 'port' => $port,
- 'type' => $type,
- 'location' => $location
- );
- }
- function getPos($haystack, $needle){ return strpos($haystack, $needle);}
-
- 函數(shù)check_proxy_is_useful($model, $proxy_info_arr = array()) {
- 全域$params, $config;
- if($model == 'single') {
- $proxy_port = intval(trim($params['port']));
- $check_proxy_url = $config['verify_url'];
- $proxy_time_out = intval(trim($params['timeout']));
- $retry = intval(trim($params['retry']));
- $proxy_ip = trim($params['ip']);
- $proxy = 新代理( $proxy_ip, $proxy_port, $check_proxy_url, $proxy_time_out, $retry );
- // 成功返回string success,失敗返回boolean false
- $result = $proxy -> check_proxy();
- //var_dump($result);
- $proxy_str_success = ''.$proxy_ip.':'.$proxy_port.'@'.'HTTP代理驗證成功! ';
- $proxy_str_failed = ''.$proxy_ip.':'.$proxy_port.'@'.'HTTP代理驗證失敗! ';
- 回傳$result !== false ? $proxy_str_success : $proxy_str_failed;
- } elseif ($model == 'collect') {
- $proxy_port = intval(triminfo_ ['port']));
- $check_proxy_url = $config['verify_url'];
- $proxy_time_out = intval(trim($params['timeout']));
- $retry = intval($params['timeout']));
- $retry = intval( trim($params['retry']));
- $proxy_ip = trim($proxy_info_arr[ 'ip']);
- /*echo $proxy_ip.'
';
- echo $ proxy_port.'
';
- echo $check_proxy_url.'';
- echo $proxy_time_out.'
';
- echo $retry.'
' ;*/
- if(!isset($proxy) )) {
- $proxy = new proxy( $proxy_ip, $proxy_port, $check_proxy_url, $proxy_time_out, $retry );
- }
- / / 成功回傳string success,失敗回傳boolean false
- $結(jié)果= $proxy -> check_proxy();
- return $result;
- }
- }
-
- function getget_single(){
- 全域$params, $config;
- $proxy_ip = trim($params ['ip']);
- if($proxy_ip == '') {
- echo '請輸入IP!! ';
- return;
- }
- echo check_proxy_is_useful('single' );
- }
-
- function get_proxy_by_collect(){
-
- function get_proxy_by_collect(){
- 38 美元, $config; $params['url'] = trim($params['url']);
- if($params['url'] == '') {
- echo '請輸入url!';
- return;
- }
- //$url = 'http:// www.dn28.com/html/75/n-5175.html';
- $con = iconv('GBK', ' UTF-8', file_get_contents($params['url']));
- preg_match ('/
|