function http_request_get in Feeds 7

Same name and namespace in other branches

1. 8.2 libraries/http_request.inc \http_request_get()
2. 6 libraries/http_request.inc \http_request_get()
3. 7.2 libraries/http_request.inc \http_request_get()

Get the content from the given URL.

Parameters

$url: A valid URL (not only web URLs).

$username: If the URL use authentication, here you can supply the username for this.

$password: If the URL use authentication, here you can supply the password for this.

Return value

A stdClass object that describes the data downloaded from $url. The object's data property contains the actual document at the URL.

3 calls to http_request_get()

FeedsEnclosure::getContent in plugins/FeedsParser.inc

FeedsHTTPBatch::getRaw in plugins/FeedsHTTPFetcher.inc

Implements FeedsImportBatch::getRaw();

http_request_get_common_syndication in libraries/http_request.inc

Discover RSS or atom feeds at the given URL. If document in given URL is an HTML document, function attempts to discover RSS or Atom feeds.

File

libraries/http_request.inc, line 86

Download via HTTP.

Code

function http_request_get($url, $username = NULL, $password = NULL, $accept_invalid_cert = FALSE) {

  // Intra-pagedownload cache, avoid to download the same content twice within one page download (it's possible, compatible and parse calls).
  static $download_cache = array();
  if (isset($download_cache[$url])) {
    return $download_cache[$url];
  }
  $has_etag = FALSE;
  $curl = http_request_use_curl();

  // Only download and parse data if really needs refresh.
  // Based on "Last-Modified" and "If-Modified-Since".
  $headers = array();
  if ($cache = cache_get('feeds_http_download_' . md5($url))) {
    $last_result = $cache->data;
    $last_headers = $last_result->headers;
    $has_etag = TRUE;
    if (!empty($last_headers['ETag'])) {
      if ($curl) {
        $headers[] = 'If-None-Match: ' . $last_headers['ETag'];
      }
      else {
        $headers['If-None-Match'] = $last_headers['ETag'];
      }
    }
    if (!empty($last_headers['Last-Modified'])) {
      if ($curl) {
        $headers[] = 'If-Modified-Since: ' . $last_headers['Last-Modified'];
      }
      else {
        $headers['If-Modified-Since'] = $last_headers['Last-Modified'];
      }
    }
    if (!empty($username) && !$curl) {
      $headers['Authorization'] = 'Basic ' . base64_encode("{$username}:{$password}");
    }
  }
  if ($curl) {
    $headers[] = 'User-Agent: Drupal (+http://drupal.org/)';
    $result = new stdClass();

    // Only download via cURL if we can validate the scheme to be either http or
    // https.
    // Validate in PHP, CURLOPT_PROTOCOLS is only supported with cURL 7.19.4
    $uri = parse_url($url);
    if (isset($uri['scheme']) && $uri['scheme'] != 'http' && $uri['scheme'] != 'https') {
      $result->error = 'invalid schema ' . $uri['scheme'];
      $result->code = -1003;

      // This corresponds to drupal_http_request()
    }
    else {
      $download = curl_init($url);
      curl_setopt($download, CURLOPT_FOLLOWLOCATION, TRUE);
      if (!empty($username)) {
        curl_setopt($download, CURLOPT_USERPWD, "{$username}:{$password}");
      }
      curl_setopt($download, CURLOPT_HTTPHEADER, $headers);
      curl_setopt($download, CURLOPT_HEADER, TRUE);
      curl_setopt($download, CURLOPT_RETURNTRANSFER, TRUE);
      curl_setopt($download, CURLOPT_ENCODING, '');
      curl_setopt($download, CURLOPT_TIMEOUT, variable_get('http_request_timeout', 15));
      if ($accept_invalid_cert) {
        curl_setopt($download, CURLOPT_SSL_VERIFYPEER, 0);
      }
      $header = '';
      $data = curl_exec($download);
      if (curl_error($download)) {
        throw new HRCurlException(t('cURL error (@code) @error for @url', array(
          '@code' => curl_errno($download),
          '@error' => curl_error($download),
          '@url' => $url,
        )), curl_errno($download));
      }
      $header_size = curl_getinfo($download, CURLINFO_HEADER_SIZE);
      $header = substr($data, 0, $header_size - 1);
      $result->data = substr($data, $header_size);
      $header_lines = preg_split("/\r\n|\n|\r/", $header);
      $result->headers = array();
      array_shift($header_lines);

      // skip HTTP response status
      while ($line = trim(array_shift($header_lines))) {
        list($header, $value) = explode(':', $line, 2);
        if (isset($result->headers[$header]) && $header == 'Set-Cookie') {

          // RFC 2109: the Set-Cookie response header comprises the token Set-
          // Cookie:, followed by a comma-separated list of one or more cookies.
          $result->headers[$header] .= ',' . trim($value);
        }
        else {
          $result->headers[$header] = trim($value);
        }
      }
      $result->code = curl_getinfo($download, CURLINFO_HTTP_CODE);
      curl_close($download);
    }
  }
  else {
    $result = drupal_http_request($url, array(
      'headers' => $headers,
    ));
  }
  $result->code = isset($result->code) ? $result->code : 200;

  // In case of 304 Not Modified try to return cached data.
  if ($result->code == 304) {
    if (isset($last_result)) {
      $last_result->from_cache = TRUE;
      return $last_result;
    }
    else {

      // It's a tragedy, this file must exist and contain good data.
      // In this case, clear cache and repeat.
      cache_clear_all('feeds_http_download_' . md5($url), 'cache');
      return http_request_get($url, $username, $password);
    }
  }
  if (!isset($result->headers) || !isset($result->headers['ETag']) || !isset($result->headers['Last-Modified'])) {
    $result->headers = isset($result->headers) ? $result->headers : array();
    $result->headers['ETag'] = isset($result->headers['ETag']) ? $result->headers['ETag'] : '';
    $result->headers['Last-Modified'] = isset($result->headers['Last-Modified']) ? $result->headers['Last-Modified'] : '';
  }

  // Set caches.
  cache_set('feeds_http_download_' . md5($url), $result);
  $download_cache[$url] = $result;
  return $result;
}