|
|
@@ -1,1162 +0,0 @@
|
|
|
-<?php
|
|
|
-/* HTTP Retriever
|
|
|
- * Version v1.1.10
|
|
|
- * Copyright 2004-2007, Steve Blinch
|
|
|
- * http://code.blitzaffe.com
|
|
|
- * ============================================================================
|
|
|
- *
|
|
|
- * DESCRIPTION
|
|
|
- *
|
|
|
- * Provides a pure-PHP implementation of an HTTP v1.1 client, including support
|
|
|
- * for chunked transfer encoding and user agent spoofing. Both GET and POST
|
|
|
- * requests are supported.
|
|
|
- *
|
|
|
- * This can be used in place of something like CURL or WGET for HTTP requests.
|
|
|
- * Native SSL (HTTPS) requests are also supported if the OpenSSL extension is
|
|
|
- * installed under PHP v4.3.0 or greater.
|
|
|
- *
|
|
|
- * If native SSL support is not available, the class will also check for the
|
|
|
- * CURL extension; if it's installed, it will transparently be used for SSL
|
|
|
- * (HTTPS) requests.
|
|
|
- *
|
|
|
- * If neither native SSL support nor the CURL extension are available, and
|
|
|
- * libcurlemu (a CURL emulation library available from our web site) is found,
|
|
|
- * the class will also check for the CURL console binary (usually in
|
|
|
- * /usr/bin/curl); if it's installed, it will transparently be used for SSL
|
|
|
- * requests.
|
|
|
- *
|
|
|
- * In short, if it's possible to make an HTTP/HTTPS request from your server,
|
|
|
- * this class can most likely do it.
|
|
|
- *
|
|
|
- *
|
|
|
- * HISTORY
|
|
|
- *
|
|
|
- * 1.1.10 (13-Feb-2006)
|
|
|
- * - Fixed bug wherein libcurlemu may not be correctly included when
|
|
|
- * needed.
|
|
|
- * - Fixed bug wherein stream read timeouts may not be recognized
|
|
|
- * - Adjusted timeout handling code to better handle timeout conditions
|
|
|
- * - Added intelligent caching support
|
|
|
- * - Caching is now better-handled for high-volume requests
|
|
|
- * - Added postprocessing callback support
|
|
|
- * - Improved redirect support
|
|
|
- * - Fixed bug in which POST requests couldn't use GET-style query strings
|
|
|
- * - Added header cleanup between requests
|
|
|
- * - Added partial proxy support via $http->curl_proxy (only useable when
|
|
|
- * $http->force_curl is TRUE; internal support not yet implemented)
|
|
|
- *
|
|
|
- *
|
|
|
- * 1.1.9 (11-Oct-2006)
|
|
|
- * - Added set_transfer_display() and default_transfer_callback()
|
|
|
- * methods for transfer progress tracking
|
|
|
- * - Suppressed possible "fatal protocol error" when remote SSL server
|
|
|
- * closes the connection early
|
|
|
- * - Added get_content_type() method
|
|
|
- * - make_query_string() now handles arrays
|
|
|
- *
|
|
|
- * 1.1.8 (19-Jun-2006)
|
|
|
- * - Added set_progress_display() and default_progress_callback()
|
|
|
- * methods for debug output
|
|
|
- * - Added support for relative URLs in HTTP redirects
|
|
|
- * - Added cookie support (sending and receiving)
|
|
|
- * - Numerous bug fixes
|
|
|
- *
|
|
|
- * 1.1.7 (18-Apr-2006)
|
|
|
- * - Added support for automatically following HTTP redirects
|
|
|
- * - Added ::get_error() method to get any available error message (be
|
|
|
- * it an HTTP result error or an internal/connection error)
|
|
|
- * - Added ::cache_hit variable to determine whether the page was cached
|
|
|
- *
|
|
|
- * 1.1.6 (04-Mar-2006)
|
|
|
- * - Added stream_timeout class variable.
|
|
|
- * - Added progress_callback class variable.
|
|
|
- * - Added support for braindead servers that ignore Connection: close
|
|
|
- *
|
|
|
- *
|
|
|
- * EXAMPLE
|
|
|
- *
|
|
|
- * // HTTPRetriever usage example
|
|
|
- * require_once("class_HTTPRetriever.php");
|
|
|
- * $http = &new HTTPRetriever();
|
|
|
- *
|
|
|
- *
|
|
|
- * // Example GET request:
|
|
|
- * // ----------------------------------------------------------------------------
|
|
|
- * $keyword = "blitzaffe code"; // search Google for this keyword
|
|
|
- * if (!$http->get("http://www.google.com/search?hl=en&q=%22".urlencode($keyword)."%22&btnG=Search&meta=")) {
|
|
|
- * echo "HTTP request error: #{$http->result_code}: {$http->result_text}";
|
|
|
- * return false;
|
|
|
- * }
|
|
|
- * echo "HTTP response headers:<br><pre>";
|
|
|
- * var_dump($http->response_headers);
|
|
|
- * echo "</pre><br>";
|
|
|
- *
|
|
|
- * echo "Page content:<br><pre>";
|
|
|
- * echo $http->response;
|
|
|
- * echo "</pre>";
|
|
|
- * // ----------------------------------------------------------------------------
|
|
|
- *
|
|
|
- *
|
|
|
- * // Example POST request:
|
|
|
- * // ----------------------------------------------------------------------------
|
|
|
- * $keyword = "blitzaffe code"; // search Google for this keyword
|
|
|
- * $values = array(
|
|
|
- * "hl"=>"en",
|
|
|
- * "q"=>"%22".urlencode($keyword)."%22",
|
|
|
- * "btnG"=>"Search",
|
|
|
- * "meta"=>""
|
|
|
- * );
|
|
|
- * // Note: This example is just to demonstrate the POST equivalent of the GET
|
|
|
- * // example above; running this script will return a 501 Not Implemented, as
|
|
|
- * // Google does not support POST requests.
|
|
|
- * if (!$http->post("http://www.google.com/search",$http->make_query_string($values))) {
|
|
|
- * echo "HTTP request error: #{$http->result_code}: {$http->result_text}";
|
|
|
- * return false;
|
|
|
- * }
|
|
|
- * echo "HTTP response headers:<br><pre>";
|
|
|
- * var_dump($http->response_headers);
|
|
|
- * echo "</pre><br>";
|
|
|
- *
|
|
|
- * echo "Page content:<br><pre>";
|
|
|
- * echo $http->response;
|
|
|
- * echo "</pre>";
|
|
|
- * // ----------------------------------------------------------------------------
|
|
|
- *
|
|
|
- *
|
|
|
- * LICENSE
|
|
|
- *
|
|
|
- * This script is free software; you can redistribute it and/or modify it under the
|
|
|
- * terms of the GNU General Public License as published by the Free Software
|
|
|
- * Foundation; either version 2 of the License, or (at your option) any later
|
|
|
- * version.
|
|
|
- *
|
|
|
- * This script is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
|
- * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
|
|
- * details.
|
|
|
- *
|
|
|
- * You should have received a copy of the GNU General Public License along
|
|
|
- * with this script; if not, write to the Free Software Foundation, Inc.,
|
|
|
- * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
- */
|
|
|
-
|
|
|
-// define user agent ID's
|
|
|
-define("UA_EXPLORER",0);
|
|
|
-define("UA_MOZILLA",1);
|
|
|
-define("UA_FIREFOX",2);
|
|
|
-define("UA_OPERA",3);
|
|
|
-
|
|
|
-// define progress message severity levels
|
|
|
-define('HRP_DEBUG',0);
|
|
|
-define('HRP_INFO',1);
|
|
|
-define('HRP_ERROR',2);
|
|
|
-
|
|
|
-if (!defined("CURL_PATH")) define("CURL_PATH","/usr/bin/curl");
|
|
|
-
|
|
|
-// if the CURL extension is not loaded, but the CURL Emulation Library is found, try
|
|
|
-// to load it
|
|
|
-if (!extension_loaded("curl") && !defined('HTTPR_NO_REDECLARE_CURL') ) {
|
|
|
- foreach (array(dirname(__FILE__)."/",dirname(__FILE__)."/libcurlemu/") as $k=>$libcurlemupath) {
|
|
|
- $libcurlemuinc = $libcurlemupath.'libcurlemu.inc.php';
|
|
|
- if (is_readable($libcurlemuinc)) require_once($libcurlemuinc);
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-class HTTPRetriever {
|
|
|
-
|
|
|
- // Constructor
|
|
|
- function HTTPRetriever() {
|
|
|
- // default HTTP headers to send with all requests
|
|
|
- $this->headers = array(
|
|
|
- "Referer"=>"",
|
|
|
- "User-Agent"=>"HTTPRetriever/1.0",
|
|
|
- "Connection"=>"close"
|
|
|
- );
|
|
|
-
|
|
|
- // HTTP version (has no effect if using CURL)
|
|
|
- $this->version = "1.1";
|
|
|
-
|
|
|
- // Normally, CURL is only used for HTTPS requests; setting this to
|
|
|
- // TRUE will force CURL for HTTP requests as well. Not recommended.
|
|
|
- $this->force_curl = false;
|
|
|
-
|
|
|
- // If you don't want to use CURL at all, set this to TRUE.
|
|
|
- $this->disable_curl = false;
|
|
|
-
|
|
|
- // If HTTPS request return an error message about SSL certificates in
|
|
|
- // $this->error and you don't care about security, set this to TRUE
|
|
|
- $this->insecure_ssl = false;
|
|
|
-
|
|
|
- // Set the maximum time to wait for a connection
|
|
|
- $this->connect_timeout = 15;
|
|
|
-
|
|
|
- // Set the maximum time to allow a transfer to run, or 0 to disable.
|
|
|
- $this->max_time = 0;
|
|
|
-
|
|
|
- // Set the maximum time for a socket read/write operation, or 0 to disable.
|
|
|
- $this->stream_timeout = 0;
|
|
|
-
|
|
|
- // If you're making an HTTPS request to a host whose SSL certificate
|
|
|
- // doesn't match its domain name, AND YOU FULLY UNDERSTAND THE
|
|
|
- // SECURITY IMPLICATIONS OF IGNORING THIS PROBLEM, set this to TRUE.
|
|
|
- $this->ignore_ssl_hostname = false;
|
|
|
-
|
|
|
- // If TRUE, the get() and post() methods will close the connection
|
|
|
- // and return immediately after receiving the HTTP result code
|
|
|
- $this->result_close = false;
|
|
|
-
|
|
|
- // If set to a positive integer value, retrieved pages will be cached
|
|
|
- // for this number of seconds. Any subsequent calls within the cache
|
|
|
- // period will return the cached page, without contacting the remote
|
|
|
- // server.
|
|
|
- $this->caching = false;
|
|
|
-
|
|
|
- // If TRUE and $this->caching is not false, retrieved pages/files will be
|
|
|
- // cached only if they appear to be static.
|
|
|
- $this->caching_intelligent = false;
|
|
|
-
|
|
|
- // If TRUE, cached files will be stored in subdirectories corresponding
|
|
|
- // to the first 2 letters of the hash filename
|
|
|
- $this->caching_highvolume = false;
|
|
|
-
|
|
|
- // If $this->caching is enabled, this specifies the folder under which
|
|
|
- // cached pages are saved.
|
|
|
- $this->cache_path = '/tmp/';
|
|
|
-
|
|
|
- // Set these to perform basic HTTP authentication
|
|
|
- $this->auth_username = '';
|
|
|
- $this->auth_password = '';
|
|
|
-
|
|
|
- // Optionally set this to a valid callback method to have HTTPRetriever
|
|
|
- // provide page preprocessing capabilities to your script. If set, this
|
|
|
- // method should accept two arguments: an object representing an instance
|
|
|
- // of HTTPRetriever, and a string containing the page contents
|
|
|
- $this->page_preprocessor = null;
|
|
|
-
|
|
|
- // Optionally set this to a valid callback method to have HTTPRetriever
|
|
|
- // provide progress messages. Your callback must accept 2 parameters:
|
|
|
- // an integer representing the severity (0=debug, 1=information, 2=error),
|
|
|
- // and a string representing the progress message
|
|
|
- $this->progress_callback = null;
|
|
|
-
|
|
|
- // Optionally set this to a valid callback method to have HTTPRetriever
|
|
|
- // provide bytes-transferred messages. Your callbcak must accept 2
|
|
|
- // parameters: an integer representing the number of bytes transferred,
|
|
|
- // and an integer representing the total number of bytes expected (or
|
|
|
- // -1 if unknown).
|
|
|
- $this->transfer_callback = null;
|
|
|
-
|
|
|
- // Set this to TRUE if you HTTPRetriever to transparently follow HTTP
|
|
|
- // redirects (code 301, 302, 303, and 307). Optionally set this to a
|
|
|
- // numeric value to limit the maximum number of redirects to the specified
|
|
|
- // value. (Redirection loops are detected automatically.)
|
|
|
- // Note that non-GET/HEAD requests will NOT be redirected except on code
|
|
|
- // 303, as per HTTP standards.
|
|
|
- $this->follow_redirects = false;
|
|
|
- }
|
|
|
-
|
|
|
- // Send an HTTP GET request to $url; if $ipaddress is specified, the
|
|
|
- // connection will be made to the selected IP instead of resolving the
|
|
|
- // hostname in $url.
|
|
|
- //
|
|
|
- // If $cookies is set, it should be an array in one of two formats.
|
|
|
- //
|
|
|
- // Either: $cookies[ 'cookiename' ] = array (
|
|
|
- // '/path/'=>array(
|
|
|
- // 'expires'=>time(),
|
|
|
- // 'domain'=>'yourdomain.com',
|
|
|
- // 'value'=>'cookievalue'
|
|
|
- // )
|
|
|
- // );
|
|
|
- //
|
|
|
- // Or, a more simplified format:
|
|
|
- // $cookies[ 'cookiename' ] = 'value';
|
|
|
- //
|
|
|
- // The former format will automatically check to make sure that the path, domain,
|
|
|
- // and expiration values match the HTTP request, and will only send the cookie if
|
|
|
- // they do match. The latter will force the cookie to be set for the HTTP request
|
|
|
- // unconditionally.
|
|
|
- //
|
|
|
- function get($url,$ipaddress = false,$cookies = false) {
|
|
|
- $this->method = "GET";
|
|
|
- $this->post_data = "";
|
|
|
- $this->connect_ip = $ipaddress;
|
|
|
- return $this->_execute_request($url,$cookies);
|
|
|
- }
|
|
|
-
|
|
|
- // Send an HTTP POST request to $url containing the POST data $data. See ::get()
|
|
|
- // for a description of the remaining arguments.
|
|
|
- function post($url,$data="",$ipaddress = false,$cookies = false) {
|
|
|
- $this->method = "POST";
|
|
|
- $this->post_data = $data;
|
|
|
- $this->connect_ip = $ipaddress;
|
|
|
- return $this->_execute_request($url,$cookies);
|
|
|
- }
|
|
|
-
|
|
|
- // Send an HTTP HEAD request to $url. See ::get() for a description of the arguments.
|
|
|
- function head($url,$ipaddress = false,$cookies = false) {
|
|
|
- $this->method = "HEAD";
|
|
|
- $this->post_data = "";
|
|
|
- $this->connect_ip = $ipaddress;
|
|
|
- return $this->_execute_request($url,$cookies);
|
|
|
- }
|
|
|
-
|
|
|
- // send an alternate (non-GET/POST) HTTP request to $url
|
|
|
- function custom($method,$url,$data="",$ipaddress = false,$cookies = false) {
|
|
|
- $this->method = $method;
|
|
|
- $this->post_data = $data;
|
|
|
- $this->connect_ip = $ipaddress;
|
|
|
- return $this->_execute_request($url,$cookies);
|
|
|
- }
|
|
|
-
|
|
|
- function array_to_query($arrayname,$arraycontents) {
|
|
|
- $output = "";
|
|
|
- foreach ($arraycontents as $key=>$value) {
|
|
|
- if (is_array($value)) {
|
|
|
- $output .= $this->array_to_query(sprintf('%s[%s]',$arrayname,urlencode($key)),$value);
|
|
|
- } else {
|
|
|
- $output .= sprintf('%s[%s]=%s&',$arrayname,urlencode($key),urlencode($value));
|
|
|
- }
|
|
|
- }
|
|
|
- return $output;
|
|
|
- }
|
|
|
-
|
|
|
- // builds a query string from the associative array array $data;
|
|
|
- // returns a string that can be passed to $this->post()
|
|
|
- function make_query_string($data) {
|
|
|
- $output = "";
|
|
|
- if (is_array($data)) {
|
|
|
- foreach ($data as $name=>$value) {
|
|
|
- if (is_array($value)) {
|
|
|
- $output .= $this->array_to_query(urlencode($name),$value);
|
|
|
- } elseif (is_scalar($value)) {
|
|
|
- $output .= urlencode($name)."=".urlencode($value)."&";
|
|
|
- } else {
|
|
|
- $output .= urlencode($name)."=".urlencode(serialize($value)).'&';
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- return substr($output,0,strlen($output)-1);
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- // this is pretty limited... but really, if you're going to spoof you UA, you'll probably
|
|
|
- // want to use a Windows OS for the spoof anyway
|
|
|
- //
|
|
|
- // if you want to set the user agent to a custom string, just assign your string to
|
|
|
- // $this->headers["User-Agent"] directly
|
|
|
- function set_user_agent($agenttype,$agentversion,$windowsversion) {
|
|
|
- $useragents = array(
|
|
|
- "Mozilla/4.0 (compatible; MSIE %agent%; Windows NT %os%)", // IE
|
|
|
- "Mozilla/5.0 (Windows; U; Windows NT %os%; en-US; rv:%agent%) Gecko/20040514", // Moz
|
|
|
- "Mozilla/5.0 (Windows; U; Windows NT %os%; en-US; rv:1.7) Gecko/20040803 Firefox/%agent%", // FFox
|
|
|
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT %os%) Opera %agent% [en]", // Opera
|
|
|
- );
|
|
|
- $agent = $useragents[$agenttype];
|
|
|
- $this->headers["User-Agent"] = str_replace(array("%agent%","%os%"),array($agentversion,$windowsversion),$agent);
|
|
|
- }
|
|
|
-
|
|
|
- // this isn't presently used as it's now handled inline by the request parser
|
|
|
- function remove_chunkiness() {
|
|
|
- $remaining = $this->response;
|
|
|
- $this->response = "";
|
|
|
-
|
|
|
- while ($remaining) {
|
|
|
- $hexlen = strpos($remaining,"\r");
|
|
|
- $chunksize = substr($remaining,0,$hexlen);
|
|
|
- $argstart = strpos($chunksize,';');
|
|
|
- if ($argstart!==false) $chunksize = substr($chunksize,0,$argstart);
|
|
|
- $chunksize = (int) @hexdec($chunksize);
|
|
|
-
|
|
|
- $this->response .= substr($remaining,$hexlen+2,$chunksize);
|
|
|
- $remaining = substr($remaining,$hexlen+2+$chunksize+2);
|
|
|
-
|
|
|
- if (!$chunksize) {
|
|
|
- // either we're done, or something's borked... exit
|
|
|
- $this->response .= $remaining;
|
|
|
- return;
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // (internal) store a page in the cache
|
|
|
- function _cache_store($token,$url) {
|
|
|
-
|
|
|
- if ($this->caching_intelligent) {
|
|
|
- $urlinfo = parse_url($url);
|
|
|
- if ($this->method=='POST') {
|
|
|
- $this->progress(HRP_DEBUG,"POST request; not caching");
|
|
|
- return;
|
|
|
- } else if (strlen($urlinfo['query'])) {
|
|
|
- $this->progress(HRP_DEBUG,"Request used query string; not caching");
|
|
|
- return;
|
|
|
- } else {
|
|
|
- $this->progress(HRP_DEBUG,"Request appears to be static and cacheable");
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- $values = array(
|
|
|
- "stats"=>$this->stats,
|
|
|
- "result_code"=>$this->result_code,
|
|
|
- "result_text"=>$this->result_text,
|
|
|
- "version"=>$this->version,
|
|
|
- "response"=>$this->response,
|
|
|
- "response_headers"=>$this->response_headers,
|
|
|
- "response_cookies"=>$this->response_cookies,
|
|
|
- "raw_response"=>$this->raw_response,
|
|
|
- );
|
|
|
- $values = serialize($values);
|
|
|
-
|
|
|
- $cache_dir = $this->cache_path;
|
|
|
- if (substr($cache_dir,-1)!='/') $cache_dir .= '/';
|
|
|
-
|
|
|
- if ($this->caching_highvolume) {
|
|
|
- $cache_dir .= substr($token,0,2) . '/';
|
|
|
- if (!is_dir($cache_dir)) @mkdir($cache_dir);
|
|
|
- }
|
|
|
-
|
|
|
- $filename = $cache_dir.$token.'.tmp';
|
|
|
-
|
|
|
- $fp = @fopen($filename,"w");
|
|
|
- if (!$fp) {
|
|
|
- $this->progress(HRP_DEBUG,"Unable to create cache file");
|
|
|
- return false;
|
|
|
- }
|
|
|
- fwrite($fp,$values);
|
|
|
- fclose($fp);
|
|
|
-
|
|
|
- $this->progress(HRP_DEBUG,"HTTP response stored to cache");
|
|
|
- }
|
|
|
-
|
|
|
- // (internal) fetch a page from the cache
|
|
|
- function _cache_fetch($token) {
|
|
|
- $this->cache_hit = false;
|
|
|
- $this->progress(HRP_DEBUG,"Checking for cached page value");
|
|
|
-
|
|
|
- $cache_dir = $this->cache_path;
|
|
|
- if (substr($cache_dir,-1)!='/') $cache_dir .= '/';
|
|
|
-
|
|
|
- if ($this->caching_highvolume) $cache_dir .= substr($token,0,2) . '/';
|
|
|
-
|
|
|
- $filename = $cache_dir.$token.'.tmp';
|
|
|
- if (!file_exists($filename)) {
|
|
|
- $this->progress(HRP_DEBUG,"Page not available in cache");
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- if (time()-filemtime($filename)>$this->caching) {
|
|
|
- $this->progress(HRP_DEBUG,"Page in cache is expired");
|
|
|
- @unlink($filename);
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- if ($values = file_get_contents($filename)) {
|
|
|
- $values = unserialize($values);
|
|
|
- if (!$values) {
|
|
|
- $this->progress(HRP_DEBUG,"Invalid cache contents");
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- $this->stats = $values["stats"];
|
|
|
- $this->result_code = $values["result_code"];
|
|
|
- $this->result_text = $values["result_text"];
|
|
|
- $this->version = $values["version"];
|
|
|
- $this->response = $values["response"];
|
|
|
- $this->response_headers = $values["response_headers"];
|
|
|
- $this->response_cookies = $values["response_cookies"];
|
|
|
- $this->raw_response = $values["raw_response"];
|
|
|
-
|
|
|
- $this->progress(HRP_DEBUG,"Page loaded from cache");
|
|
|
- $this->cache_hit = true;
|
|
|
- return true;
|
|
|
- } else {
|
|
|
- $this->progress(HRP_DEBUG,"Error reading cache file");
|
|
|
- return false;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- function parent_path($path) {
|
|
|
- if (substr($path,0,1)=='/') $path = substr($path,1);
|
|
|
- if (substr($path,-1)=='/') $path = substr($path,0,strlen($path)-1);
|
|
|
- $path = explode('/',$path);
|
|
|
- array_pop($path);
|
|
|
- return count($path) ? ('/' . implode('/',$path)) : '';
|
|
|
- }
|
|
|
-
|
|
|
- // $cookies should be an array in one of two formats.
|
|
|
- //
|
|
|
- // Either: $cookies[ 'cookiename' ] = array (
|
|
|
- // '/path/'=>array(
|
|
|
- // 'expires'=>time(),
|
|
|
- // 'domain'=>'yourdomain.com',
|
|
|
- // 'value'=>'cookievalue'
|
|
|
- // )
|
|
|
- // );
|
|
|
- //
|
|
|
- // Or, a more simplified format:
|
|
|
- // $cookies[ 'cookiename' ] = 'value';
|
|
|
- //
|
|
|
- // The former format will automatically check to make sure that the path, domain,
|
|
|
- // and expiration values match the HTTP request, and will only send the cookie if
|
|
|
- // they do match. The latter will force the cookie to be set for the HTTP request
|
|
|
- // unconditionally.
|
|
|
- //
|
|
|
- function response_to_request_cookies($cookies,$urlinfo) {
|
|
|
-
|
|
|
- // check for simplified cookie format (name=value)
|
|
|
- $cookiekeys = array_keys($cookies);
|
|
|
- if (!count($cookiekeys)) return;
|
|
|
-
|
|
|
- $testkey = array_pop($cookiekeys);
|
|
|
- if (!is_array($cookies[ $testkey ])) {
|
|
|
- foreach ($cookies as $k=>$v) $this->request_cookies[$k] = $v;
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
- // must not be simplified format, so parse as complex format:
|
|
|
- foreach ($cookies as $name=>$paths) {
|
|
|
- foreach ($paths as $path=>$values) {
|
|
|
- // make sure the cookie isn't expired
|
|
|
- if ( isset($values['expires']) && ($values['expires']<time()) ) continue;
|
|
|
-
|
|
|
- $cookiehost = $values['domain'];
|
|
|
- $requesthost = $urlinfo['host'];
|
|
|
- // make sure the cookie is valid for this host
|
|
|
- $domain_match = (
|
|
|
- ($requesthost==$cookiehost) ||
|
|
|
- (substr($requesthost,-(strlen($cookiehost)+1))=='.'.$cookiehost)
|
|
|
- );
|
|
|
-
|
|
|
- // make sure the cookie is valid for this path
|
|
|
- $cookiepath = $path; if (substr($cookiepath,-1)!='/') $cookiepath .= '/';
|
|
|
- $requestpath = $urlinfo['path']; if (substr($requestpath,-1)!='/') $requestpath .= '/';
|
|
|
- if (substr($requestpath,0,strlen($cookiepath))!=$cookiepath) continue;
|
|
|
-
|
|
|
- $this->request_cookies[$name] = $values['value'];
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // Execute the request for a particular URL, and transparently follow
|
|
|
- // HTTP redirects if enabled. If $cookies is specified, it is assumed
|
|
|
- // to be an array received from $this->response_cookies and will be
|
|
|
- // processed to determine which cookies are valid for this host/URL.
|
|
|
- function _execute_request($url,$cookies = false) {
|
|
|
- // valid codes for which we transparently follow a redirect
|
|
|
- $redirect_codes = array(301,302,303,307);
|
|
|
- // valid methods for which we transparently follow a redirect
|
|
|
- $redirect_methods = array('GET','HEAD');
|
|
|
-
|
|
|
- $request_result = false;
|
|
|
-
|
|
|
- $this->followed_redirect = false;
|
|
|
- $this->response_cookies = array();
|
|
|
- $this->cookie_headers = '';
|
|
|
-
|
|
|
- $previous_redirects = array();
|
|
|
- do {
|
|
|
- // send the request
|
|
|
- $request_result = $this->_send_request($url,$cookies);
|
|
|
- $lasturl = $url;
|
|
|
- $url = false;
|
|
|
-
|
|
|
- // see if a redirect code was received
|
|
|
- if ($this->follow_redirects && in_array($this->result_code,$redirect_codes)) {
|
|
|
-
|
|
|
- // only redirect on a code 303 or if the method was GET/HEAD
|
|
|
- if ( ($this->result_code==303) || in_array($this->method,$redirect_methods) ) {
|
|
|
-
|
|
|
- // parse the information from the OLD URL so that we can handle
|
|
|
- // relative links
|
|
|
- $oldurlinfo = parse_url($lasturl);
|
|
|
-
|
|
|
- $url = $this->response_headers['Location'];
|
|
|
-
|
|
|
- // parse the information in the new URL, and fill in any blanks
|
|
|
- // using values from the old URL
|
|
|
- $urlinfo = parse_url($url);
|
|
|
- foreach ($oldurlinfo as $k=>$v) {
|
|
|
- if (!$urlinfo[$k]) $urlinfo[$k] = $v;
|
|
|
- }
|
|
|
-
|
|
|
- // create an absolute path
|
|
|
- if (substr($urlinfo['path'],0,1)!='/') {
|
|
|
- $baseurl = $oldurlinfo['path'];
|
|
|
- if (substr($baseurl,-1)!='/') $baseurl = $this->parent_path($url) . '/';
|
|
|
- $urlinfo['path'] = $baseurl . $urlinfo['path'];
|
|
|
- }
|
|
|
-
|
|
|
- // rebuild the URL
|
|
|
- $url = $this->rebuild_url($urlinfo);
|
|
|
-
|
|
|
- $this->method = "GET";
|
|
|
- $this->post_data = "";
|
|
|
-
|
|
|
- $this->progress(HRP_INFO,'Redirected to '.$url);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if ( $url && strlen($url) ) {
|
|
|
-
|
|
|
- if (isset($previous_redirects[$url])) {
|
|
|
- $this->error = "Infinite redirection loop";
|
|
|
- $request_result = false;
|
|
|
- break;
|
|
|
- }
|
|
|
- if ( is_numeric($this->follow_redirects) && (count($previous_redirects)>$this->follow_redirects) ) {
|
|
|
- $this->error = "Exceeded redirection limit";
|
|
|
- $request_result = false;
|
|
|
- break;
|
|
|
- }
|
|
|
-
|
|
|
- $previous_redirects[$url] = true;
|
|
|
- }
|
|
|
-
|
|
|
- } while ($url && strlen($url));
|
|
|
-
|
|
|
- // clear headers that shouldn't persist across multiple requests
|
|
|
- $per_request_headers = array('Host','Content-Length');
|
|
|
- foreach ($per_request_headers as $k=>$v) unset($this->headers[$v]);
|
|
|
-
|
|
|
- if (count($previous_redirects)>1) $this->followed_redirect = array_keys($previous_redirects);
|
|
|
-
|
|
|
- return $request_result;
|
|
|
- }
|
|
|
-
|
|
|
- // private - sends an HTTP request to $url
|
|
|
- function _send_request($url,$cookies = false) {
|
|
|
- $this->progress(HRP_INFO,"Initiating {$this->method} request for $url");
|
|
|
- if ($this->caching) {
|
|
|
- $cachetoken = md5($url.'|'.$this->post_data);
|
|
|
- if ($this->_cache_fetch($cachetoken)) return true;
|
|
|
- }
|
|
|
-
|
|
|
- $time_request_start = $this->getmicrotime();
|
|
|
-
|
|
|
- $urldata = parse_url($url);
|
|
|
- $this->urldata = &$urldata;
|
|
|
- $http_host = $urldata['host'] . (isset($urldata['port']) ? ':'.$urldata['port'] : '');
|
|
|
-
|
|
|
- if (!isset($urldata["port"]) || !$urldata["port"]) $urldata["port"] = ($urldata["scheme"]=="https") ? 443 : 80;
|
|
|
- if (!isset($urldata["path"]) || !$urldata["path"]) $urldata["path"] = '/';
|
|
|
-
|
|
|
- if (!empty($urldata['user'])) $this->auth_username = $urldata['user'];
|
|
|
- if (!empty($urldata['pass'])) $this->auth_password = $urldata['pass'];
|
|
|
-
|
|
|
- //echo "Sending HTTP/{$this->version} {$this->method} request for ".$urldata["host"].":".$urldata["port"]." page ".$urldata["path"]."<br>";
|
|
|
-
|
|
|
- if ($this->version>"1.0") $this->headers["Host"] = $http_host;
|
|
|
- if ($this->method=="POST") {
|
|
|
- $this->headers["Content-Length"] = strlen($this->post_data);
|
|
|
- if (!isset($this->headers["Content-Type"])) $this->headers["Content-Type"] = "application/x-www-form-urlencoded";
|
|
|
- }
|
|
|
-
|
|
|
- if ( !empty($this->auth_username) || !empty($this->auth_password) ) {
|
|
|
- $this->headers['Authorization'] = 'Basic '.base64_encode($this->auth_username.':'.$this->auth_password);
|
|
|
- } else {
|
|
|
- unset($this->headers['Authorization']);
|
|
|
- }
|
|
|
-
|
|
|
- if (is_array($cookies)) {
|
|
|
- $this->response_to_request_cookies($cookies,$urldata);
|
|
|
- }
|
|
|
-
|
|
|
- if (!empty($urldata["query"])) $urldata["path"] .= "?".$urldata["query"];
|
|
|
- $request = $this->method." ".$urldata["path"]." HTTP/".$this->version."\r\n";
|
|
|
- $request .= $this->build_headers();
|
|
|
- $request .= $this->post_data;
|
|
|
-
|
|
|
- $this->response = "";
|
|
|
-
|
|
|
- // clear headers that shouldn't persist across multiple requests
|
|
|
- // (we can do this here as we've already built the request, including headers, above)
|
|
|
- $per_request_headers = array('Host','Content-Length');
|
|
|
- foreach ($per_request_headers as $k=>$v) unset($this->headers[$v]);
|
|
|
-
|
|
|
- // Native SSL support requires the OpenSSL extension, and was introduced in PHP 4.3.0
|
|
|
- $php_ssl_support = extension_loaded("openssl") && version_compare(phpversion(),"4.3.0")>=0;
|
|
|
-
|
|
|
- // if this is a plain HTTP request, or if it's an HTTPS request and OpenSSL support is available,
|
|
|
- // natively perform the HTTP request
|
|
|
- if ( ( ($urldata["scheme"]=="http") || ($php_ssl_support && ($urldata["scheme"]=="https")) ) && (!$this->force_curl) ) {
|
|
|
- $curl_mode = false;
|
|
|
-
|
|
|
- $hostname = $this->connect_ip ? $this->connect_ip : $urldata['host'];
|
|
|
- if ($urldata["scheme"]=="https") $hostname = 'ssl://'.$hostname;
|
|
|
-
|
|
|
- $time_connect_start = $this->getmicrotime();
|
|
|
-
|
|
|
- $this->progress(HRP_INFO,'Opening socket connection to '.$hostname.' port '.$urldata['port']);
|
|
|
-
|
|
|
- $this->expected_bytes = -1;
|
|
|
- $this->received_bytes = 0;
|
|
|
-
|
|
|
- $fp = @fsockopen ($hostname,$urldata["port"],$errno,$errstr,$this->connect_timeout);
|
|
|
- $time_connected = $this->getmicrotime();
|
|
|
- $connect_time = $time_connected - $time_connect_start;
|
|
|
- if ($fp) {
|
|
|
- if ($this->stream_timeout) stream_set_timeout($fp,$this->stream_timeout);
|
|
|
- $this->progress(HRP_INFO,"Connected; sending request");
|
|
|
-
|
|
|
- $this->progress(HRP_DEBUG,$request);
|
|
|
- fputs ($fp, $request);
|
|
|
- $this->raw_request = $request;
|
|
|
-
|
|
|
- if ($this->stream_timeout) {
|
|
|
- $meta = socket_get_status($fp);
|
|
|
- if ($meta['timed_out']) {
|
|
|
- $this->error = "Exceeded socket write timeout of ".$this->stream_timeout." seconds";
|
|
|
- $this->progress(HRP_ERROR,$this->error);
|
|
|
- return false;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- $this->progress(HRP_INFO,"Request sent; awaiting reply");
|
|
|
-
|
|
|
- $headers_received = false;
|
|
|
- $data_length = false;
|
|
|
- $chunked = false;
|
|
|
- $iterations = 0;
|
|
|
- while (!feof($fp)) {
|
|
|
- if ($data_length>0) {
|
|
|
- $line = fread($fp,$data_length);
|
|
|
- $this->progress(HRP_DEBUG,"[DL] Got a line: [{$line}] " . gettype($line));
|
|
|
-
|
|
|
- if ($line!==false) $data_length -= strlen($line);
|
|
|
- } else {
|
|
|
- $line = @fgets($fp,10240);
|
|
|
- $this->progress(HRP_DEBUG,"[NDL] Got a line: [{$line}] " . gettype($line));
|
|
|
-
|
|
|
- if ( ($chunked) && ($line!==false) ) {
|
|
|
- $line = trim($line);
|
|
|
- if (!strlen($line)) continue;
|
|
|
-
|
|
|
- list($data_length,) = explode(';',$line,2);
|
|
|
- $data_length = (int) hexdec(trim($data_length));
|
|
|
-
|
|
|
- if ($data_length==0) {
|
|
|
- $this->progress(HRP_DEBUG,"Done");
|
|
|
- // end of chunked data
|
|
|
- break;
|
|
|
- }
|
|
|
- $this->progress(HRP_DEBUG,"Chunk length $data_length (0x$line)");
|
|
|
- continue;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if ($line===false) {
|
|
|
- $meta = socket_get_status($fp);
|
|
|
- if ($meta['timed_out']) {
|
|
|
- if ($this->stream_timeout) {
|
|
|
- $this->error = "Exceeded socket read timeout of ".$this->stream_timeout." seconds";
|
|
|
- } else {
|
|
|
- $this->error = "Exceeded default socket read timeout";
|
|
|
- }
|
|
|
- $this->progress(HRP_ERROR,$this->error);
|
|
|
- return false;
|
|
|
- } else {
|
|
|
- $this->progress(HRP_ERROR,'No data but not timed out');
|
|
|
- }
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- // check time limits if requested
|
|
|
- if ($this->max_time>0) {
|
|
|
- if ($this->getmicrotime() - $time_request_start > $this->max_time) {
|
|
|
- $this->error = "Exceeded maximum transfer time of ".$this->max_time." seconds";
|
|
|
- $this->progress(HRP_ERROR,$this->error);
|
|
|
- return false;
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- $this->response .= $line;
|
|
|
-
|
|
|
- $iterations++;
|
|
|
- if ($headers_received) {
|
|
|
- if ($time_connected>0) {
|
|
|
- $time_firstdata = $this->getmicrotime();
|
|
|
- $process_time = $time_firstdata - $time_connected;
|
|
|
- $time_connected = 0;
|
|
|
- }
|
|
|
- $this->received_bytes += strlen($line);
|
|
|
- if ($iterations % 20 == 0) {
|
|
|
- $this->update_transfer_counters();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- // some dumbass webservers don't respect Connection: close and just
|
|
|
- // leave the connection open, so we have to be diligent about
|
|
|
- // calculating the content length so we can disconnect at the end of
|
|
|
- // the response
|
|
|
- if ( (!$headers_received) && (trim($line)=="") ) {
|
|
|
- $headers_received = true;
|
|
|
- $this->progress(HRP_DEBUG,"Got headers: {$this->response}");
|
|
|
-
|
|
|
- if (preg_match('/^Content-Length: ([0-9]+)/im',$this->response,$matches)) {
|
|
|
- $data_length = (int) $matches[1];
|
|
|
- $this->progress(HRP_DEBUG,"Content length is $data_length");
|
|
|
- $this->expected_bytes = $data_length;
|
|
|
- $this->update_transfer_counters();
|
|
|
- } else {
|
|
|
- $this->progress(HRP_DEBUG,"No data length specified");
|
|
|
- }
|
|
|
- if (preg_match("/^Transfer-Encoding: chunked/im",$this->response,$matches)) {
|
|
|
- $chunked = true;
|
|
|
- $this->progress(HRP_DEBUG,"Chunked transfer encoding requested");
|
|
|
- } else {
|
|
|
- $this->progress(HRP_DEBUG,"CTE not requested");
|
|
|
- }
|
|
|
-
|
|
|
- if (preg_match_all("/^Set-Cookie: ((.*?)\=(.*?)(?:;\s*(.*))?)$/im",$this->response,$cookielist,PREG_SET_ORDER)) {
|
|
|
- foreach ($cookielist as $k=>$cookie) $this->cookie_headers .= $cookie[0]."\n";
|
|
|
-
|
|
|
- // get the path for which cookies will be valid if no path is specified
|
|
|
- $cookiepath = preg_replace('/\/{2,}/','',$urldata['path']);
|
|
|
- if (substr($cookiepath,-1)!='/') {
|
|
|
- $cookiepath = explode('/',$cookiepath);
|
|
|
- array_pop($cookiepath);
|
|
|
- $cookiepath = implode('/',$cookiepath) . '/';
|
|
|
- }
|
|
|
- // process each cookie
|
|
|
- foreach ($cookielist as $k=>$cookiedata) {
|
|
|
- list(,$rawcookie,$name,$value,$attributedata) = $cookiedata;
|
|
|
- $attributedata = explode(';',trim($attributedata));
|
|
|
- $attributes = array();
|
|
|
-
|
|
|
- $cookie = array(
|
|
|
- 'value'=>$value,
|
|
|
- 'raw'=>trim($rawcookie),
|
|
|
- );
|
|
|
- foreach ($attributedata as $k=>$attribute) {
|
|
|
- list($attrname,$attrvalue) = explode('=',trim($attribute));
|
|
|
- $cookie[$attrname] = $attrvalue;
|
|
|
- }
|
|
|
-
|
|
|
- if (!isset($cookie['domain']) || !$cookie['domain']) $cookie['domain'] = $urldata['host'];
|
|
|
- if (!isset($cookie['path']) || !$cookie['path']) $cookie['path'] = $cookiepath;
|
|
|
- if (isset($cookie['expires']) && $cookie['expires']) $cookie['expires'] = strtotime($cookie['expires']);
|
|
|
-
|
|
|
- if (!$this->validate_response_cookie($cookie,$urldata['host'])) continue;
|
|
|
-
|
|
|
- // do not store expired cookies; if one exists, unset it
|
|
|
- if ( isset($cookie['expires']) && ($cookie['expires']<time()) ) {
|
|
|
- unset($this->response_cookies[ $name ][ $cookie['path'] ]);
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- $this->response_cookies[ $name ][ $cookie['path'] ] = $cookie;
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if ($this->result_close) {
|
|
|
- if (preg_match_all("/HTTP\/([0-9\.]+) ([0-9]+) (.*?)[\r\n]/",$this->response,$matches)) {
|
|
|
- $resultcodes = $matches[2];
|
|
|
- foreach ($resultcodes as $k=>$code) {
|
|
|
- if ($code!=100) {
|
|
|
- $this->progress(HRP_INFO,'HTTP result code received; closing connection');
|
|
|
-
|
|
|
- $this->result_code = $code;
|
|
|
- $this->result_text = $matches[3][$k];
|
|
|
- fclose($fp);
|
|
|
-
|
|
|
- return ($this->result_code==200);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- if (feof($fp)) $this->progress(HRP_DEBUG,'EOF on socket');
|
|
|
- @fclose ($fp);
|
|
|
-
|
|
|
- $this->update_transfer_counters();
|
|
|
-
|
|
|
- if (is_array($this->response_cookies)) {
|
|
|
- // make sure paths are sorted in the order in which they should be applied
|
|
|
- // when setting response cookies
|
|
|
- foreach ($this->response_cookies as $name=>$paths) {
|
|
|
- ksort($this->response_cookies[$name]);
|
|
|
- }
|
|
|
- }
|
|
|
- $this->progress(HRP_INFO,'Request complete');
|
|
|
- } else {
|
|
|
- $this->error = strtoupper($urldata["scheme"])." connection to ".$hostname." port ".$urldata["port"]." failed";
|
|
|
- $this->progress(HRP_ERROR,$this->error);
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- // perform an HTTP/HTTPS request using CURL
|
|
|
- } elseif ( !$this->disable_curl && ( ($urldata["scheme"]=="https") || ($this->force_curl) ) ) {
|
|
|
- $this->progress(HRP_INFO,'Passing HTTP request for $url to CURL');
|
|
|
- $curl_mode = true;
|
|
|
- if (!$this->_curl_request($url)) return false;
|
|
|
-
|
|
|
- // unknown protocol
|
|
|
- } else {
|
|
|
- $this->error = "Unsupported protocol: ".$urldata["scheme"];
|
|
|
- $this->progress(HRP_ERROR,$this->error);
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- $this->raw_response = $this->response;
|
|
|
-
|
|
|
- $totallength = strlen($this->response);
|
|
|
-
|
|
|
- do {
|
|
|
- $headerlength = strpos($this->response,"\r\n\r\n");
|
|
|
-
|
|
|
- $response_headers = explode("\r\n",substr($this->response,0,$headerlength));
|
|
|
- $http_status = trim(array_shift($response_headers));
|
|
|
- foreach ($response_headers as $line) {
|
|
|
- list($k,$v) = explode(":",$line,2);
|
|
|
- $this->response_headers[trim($k)] = trim($v);
|
|
|
- }
|
|
|
- $this->response = substr($this->response,$headerlength+4);
|
|
|
-
|
|
|
- /* // Handled in-transfer now
|
|
|
- if (($this->response_headers['Transfer-Encoding']=="chunked") && (!$curl_mode)) {
|
|
|
- $this->remove_chunkiness();
|
|
|
- }
|
|
|
- */
|
|
|
-
|
|
|
- if (!preg_match("/^HTTP\/([0-9\.]+) ([0-9]+) (.*?)$/",$http_status,$matches)) {
|
|
|
- $matches = array("",$this->version,0,"HTTP request error");
|
|
|
- }
|
|
|
- list (,$response_version,$this->result_code,$this->result_text) = $matches;
|
|
|
-
|
|
|
- // skip HTTP result code 100 (Continue) responses
|
|
|
- } while (($this->result_code==100) && ($headerlength));
|
|
|
-
|
|
|
- // record some statistics, roughly compatible with CURL's curl_getinfo()
|
|
|
- if (!$curl_mode) {
|
|
|
- $total_time = $this->getmicrotime() - $time_request_start;
|
|
|
- $transfer_time = $total_time - $connect_time;
|
|
|
- $this->stats = array(
|
|
|
- "total_time"=>$total_time,
|
|
|
- "connect_time"=>$connect_time, // time between connection request and connection established
|
|
|
- "process_time"=>$process_time, // time between HTTP request and first data (non-headers) received
|
|
|
- "url"=>$url,
|
|
|
- "content_type"=>$this->response_headers["Content-Type"],
|
|
|
- "http_code"=>$this->result_code,
|
|
|
- "header_size"=>$headerlength,
|
|
|
- "request_size"=>$totallength,
|
|
|
- "filetime"=>strtotime($this->response_headers["Date"]),
|
|
|
- "pretransfer_time"=>$connect_time,
|
|
|
- "size_download"=>$totallength,
|
|
|
- "speed_download"=>$transfer_time > 0 ? round($totallength / $transfer_time) : 0,
|
|
|
- "download_content_length"=>$totallength,
|
|
|
- "upload_content_length"=>0,
|
|
|
- "starttransfer_time"=>$connect_time,
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- $ok = ($this->result_code==200);
|
|
|
- if ($ok) {
|
|
|
- // if a page preprocessor is defined, call it to process the page contents
|
|
|
- if (is_callable($this->page_preprocessor)) $this->response = call_user_func($this->page_preprocessor,$this,$this->response);
|
|
|
-
|
|
|
- // if caching is enabled, save the page
|
|
|
- if ($this->caching) $this->_cache_store($cachetoken,$url);
|
|
|
- }
|
|
|
-
|
|
|
- return $ok;
|
|
|
- }
|
|
|
-
|
|
|
- function validate_response_cookie($cookie,$actual_hostname) {
|
|
|
- // make sure the cookie can't be set for a TLD, eg: '.com'
|
|
|
- $cookiehost = $cookie['domain'];
|
|
|
- $p = strrpos($cookiehost,'.');
|
|
|
- if ($p===false) return false;
|
|
|
-
|
|
|
- $tld = strtolower(substr($cookiehost,$p+1));
|
|
|
- $special_domains = array("com", "edu", "net", "org", "gov", "mil", "int");
|
|
|
- $periods_required = in_array($tld,$special_domains) ? 1 : 2;
|
|
|
-
|
|
|
- $periods = substr_count($cookiehost,'.');
|
|
|
- if ($periods<$periods_required) return false;
|
|
|
-
|
|
|
- if (substr($actual_hostname,0,1)!='.') $actual_hostname = '.'.$actual_hostname;
|
|
|
- if (substr($cookiehost,0,1)!='.') $cookiehost = '.'.$cookiehost;
|
|
|
- $domain_match = (
|
|
|
- ($actual_hostname==$cookiehost) ||
|
|
|
- (substr($actual_hostname,-strlen($cookiehost))==$cookiehost)
|
|
|
- );
|
|
|
-
|
|
|
- return $domain_match;
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- function build_headers() {
|
|
|
- $headers = "";
|
|
|
- foreach ($this->headers as $name=>$value) {
|
|
|
- $value = trim($value);
|
|
|
- if (empty($value)) continue;
|
|
|
- $headers .= "{$name}: {$value}\r\n";
|
|
|
- }
|
|
|
-
|
|
|
- if (isset($this->request_cookies) && is_array($this->request_cookies)) {
|
|
|
- $cookielist = array();
|
|
|
- foreach ($this->request_cookies as $name=>$value) {
|
|
|
- $cookielist[] = "{$name}={$value}";
|
|
|
- }
|
|
|
- if (count($cookielist)) $headers .= "Cookie: ".implode('; ',$cookielist)."\r\n";
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- $headers .= "\r\n";
|
|
|
-
|
|
|
- return $headers;
|
|
|
- }
|
|
|
-
|
|
|
- // opposite of parse_url()
|
|
|
- function rebuild_url($urlinfo) {
|
|
|
- $url = $urlinfo['scheme'].'://';
|
|
|
-
|
|
|
- if ($urlinfo['user'] || $urlinfo['pass']) {
|
|
|
- $url .= $urlinfo['user'];
|
|
|
- if ($urlinfo['pass']) {
|
|
|
- if ($urlinfo['user']) $url .= ':';
|
|
|
- $url .= $urlinfo['pass'];
|
|
|
- }
|
|
|
- $url .= '@';
|
|
|
- }
|
|
|
-
|
|
|
- $url .= $urlinfo['host'];
|
|
|
- if ($urlinfo['port']) $url .= ':'.$urlinfo['port'];
|
|
|
-
|
|
|
- $url .= $urlinfo['path'];
|
|
|
-
|
|
|
- if ($urlinfo['query']) $url .= '?'.$urlinfo['query'];
|
|
|
- if ($urlinfo['fragment']) $url .= '#'.$urlinfo['fragment'];
|
|
|
-
|
|
|
- return $url;
|
|
|
- }
|
|
|
-
|
|
|
- function _replace_hostname(&$url,$new_hostname) {
|
|
|
- $parts = parse_url($url);
|
|
|
- $old_hostname = $parts['host'];
|
|
|
-
|
|
|
- $parts['host'] = $new_hostname;
|
|
|
-
|
|
|
- $url = $this->rebuild_url($parts);
|
|
|
-
|
|
|
- return $old_hostname;
|
|
|
- }
|
|
|
-
|
|
|
- function _curl_request($url) {
|
|
|
- $this->error = false;
|
|
|
-
|
|
|
- // if a direct connection IP address was specified, replace the hostname
|
|
|
- // in the URL with the IP address, and set the Host: header to the
|
|
|
- // original hostname
|
|
|
- if ($this->connect_ip) {
|
|
|
- $old_hostname = $this->_replace_hostname($url,$this->connect_ip);
|
|
|
- $this->headers["Host"] = $old_hostname;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- unset($this->headers["Content-Length"]);
|
|
|
- $headers = explode("\n",$this->build_headers());
|
|
|
-
|
|
|
- $ch = curl_init();
|
|
|
- curl_setopt($ch,CURLOPT_URL, $url);
|
|
|
- curl_setopt($ch,CURLOPT_USERAGENT, $this->headers["User-Agent"]);
|
|
|
- curl_setopt($ch,CURLOPT_HEADER, 1);
|
|
|
- curl_setopt($ch,CURLOPT_RETURNTRANSFER, 1);
|
|
|
-// curl_setopt($ch,CURLOPT_FOLLOWLOCATION, 1); // native method doesn't support this yet, so it's disabled for consistency
|
|
|
- curl_setopt($ch,CURLOPT_TIMEOUT, 10);
|
|
|
- if ($this->curl_proxy) {
|
|
|
- curl_setopt($ch,CURLOPT_PROXY,$this->curl_proxy);
|
|
|
- }
|
|
|
- curl_setopt($ch,CURLOPT_HTTPHEADER, $headers);
|
|
|
-
|
|
|
- if ($this->method=="POST") {
|
|
|
- curl_setopt($ch,CURLOPT_POST,1);
|
|
|
- curl_setopt($ch,CURLOPT_POSTFIELDS,$this->post_data);
|
|
|
- }
|
|
|
- if ($this->insecure_ssl) {
|
|
|
- curl_setopt($ch,CURLOPT_SSL_VERIFYPEER,0);
|
|
|
- }
|
|
|
- if ($this->ignore_ssl_hostname) {
|
|
|
- curl_setopt($ch,CURLOPT_SSL_VERIFYHOST,1);
|
|
|
- }
|
|
|
-
|
|
|
- $this->response = curl_exec ($ch);
|
|
|
- if (curl_errno($ch)!=0) {
|
|
|
- $this->error = "CURL error #".curl_errno($ch).": ".curl_error($ch);
|
|
|
- }
|
|
|
-
|
|
|
- $this->stats = curl_getinfo($ch);
|
|
|
- curl_close($ch);
|
|
|
-
|
|
|
- return ($this->error === false);
|
|
|
- }
|
|
|
-
|
|
|
- function progress($level,$msg) {
|
|
|
- if (is_callable($this->progress_callback)) call_user_func($this->progress_callback,$level,$msg);
|
|
|
- }
|
|
|
-
|
|
|
- // Gets any available HTTPRetriever error message (including both internal
|
|
|
- // errors and HTTP errors)
|
|
|
- function get_error() {
|
|
|
- return $this->error ? $this->error : 'HTTP ' . $this->result_code.': '.$this->result_text;
|
|
|
- }
|
|
|
-
|
|
|
- function get_content_type() {
|
|
|
- if (!$ctype = $this->response_headers['Content-Type']) {
|
|
|
- $ctype = $this->response_headers['Content-type'];
|
|
|
- }
|
|
|
- list($ctype,) = explode(';',$ctype);
|
|
|
-
|
|
|
- return strtolower($ctype);
|
|
|
- }
|
|
|
-
|
|
|
- function update_transfer_counters() {
|
|
|
- if (is_callable($this->transfer_callback)) call_user_func($this->transfer_callback,$this->received_bytes,$this->expected_bytes);
|
|
|
- }
|
|
|
-
|
|
|
- function set_transfer_display($enabled = true) {
|
|
|
- if ($enabled) {
|
|
|
- $this->transfer_callback = array(&$this,'default_transfer_callback');
|
|
|
- } else {
|
|
|
- unset($this->transfer_callback);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- function set_progress_display($enabled = true) {
|
|
|
- if ($enabled) {
|
|
|
- $this->progress_callback = array(&$this,'default_progress_callback');
|
|
|
- } else {
|
|
|
- unset($this->progress_callback);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- function default_progress_callback($severity,$message) {
|
|
|
- $severities = array(
|
|
|
- HRP_DEBUG=>'debug',
|
|
|
- HRP_INFO=>'info',
|
|
|
- HRP_ERROR=>'error',
|
|
|
- );
|
|
|
-
|
|
|
- echo date('Y-m-d H:i:sa').' ['.$severities[$severity].'] '.$message."\n";
|
|
|
- flush();
|
|
|
- }
|
|
|
-
|
|
|
- function default_transfer_callback($transferred,$expected) {
|
|
|
- $msg = "Transferred " . round($transferred/1024,1);
|
|
|
- if ($expected>=0) $msg .= "/" . round($expected/1024,1);
|
|
|
- $msg .= "KB";
|
|
|
- if ($expected>0) $msg .= " (".round($transferred*100/$expected,1)."%)";
|
|
|
- echo date('Y-m-d H:i:sa')." $msg\n";
|
|
|
- flush();
|
|
|
- }
|
|
|
-
|
|
|
- function getmicrotime() {
|
|
|
- list($usec, $sec) = explode(" ",microtime());
|
|
|
- return ((float)$usec + (float)$sec);
|
|
|
- }
|
|
|
-}
|
|
|
-?>
|