function _s3fs_refresh_cache in S3 File System 7.3

Same name and namespace in other branches

1. 7 s3fs.module \_s3fs_refresh_cache()
2. 7.2 s3fs.module \_s3fs_refresh_cache()

Refreshes the metadata cache.

Iterates over the full list of objects in the s3fs_root_folder within S3 bucket (or the entire bucket, if no root folder has been set), caching their metadata in the database.

It then caches the ancestor folders for those files, since folders are not normally stored as actual objects in S3.

Parameters

array $config: An s3fs configuration array.

4 calls to _s3fs_refresh_cache()

drush_s3fs_refresh_cache in ./s3fs.drush.inc

Refreshes the file metadata cache.

s3fs.test in tests/s3fs.test

s3fs_update_7000 in ./s3fs.install

Allow large filesize values in the S3 File Metadata cache.

_s3fs_refresh_cache_submit in ./s3fs.admin.inc

Submit callback for the "Refresh file metadata cache" button.

File

./s3fs.module, line 378

Hook implementations and other primary functionality for S3 File System.

Code

function _s3fs_refresh_cache($config) {

  // Bomb out with an error if our configuration settings are invalid.
  if (!_s3fs_validate_config($config)) {
    form_set_error('s3fs_refresh_cache][refresh', t('Unable to validate S3 configuration settings.'));
    return;
  }
  if (function_exists('drush_log')) {
    drush_log('Getting Amazon S3 client...');
  }
  $s3 = _s3fs_get_amazons3_client($config);

  // Set up the iterator that will loop over all the objects in the bucket.
  $iterator_args = array(
    'Bucket' => $config['bucket'],
  );
  if (!empty($config['root_folder'])) {

    // If the root_folder option has been set, retrieve from S3 only those files
    // which reside in the root folder.
    $iterator_args['Prefix'] = "{$config['root_folder']}/";
  }

  // Determine if object versions should be included/excluded
  // as part of the ListObjects query.
  $op_name = _s3fs_get_setting('use_versioning') ? 'ListObjectVersions' : 'ListObjects';
  $iterator = $s3
    ->getPaginator($op_name, $iterator_args);
  if (function_exists('drush_log')) {
    drush_log('Creating temporary tables...');
  }

  // Create the temp table, into which all the refreshed data will be written.
  // After the full refresh is complete, the temp table will be swapped with
  // the real one.
  module_load_install('s3fs');
  $schema = s3fs_schema();
  try {
    db_create_table('s3fs_file_temp', $schema['s3fs_file']);
  } catch (DatabaseSchemaObjectExistsException $e) {

    // The table already exists, so we can simply truncate it to start fresh.
    db_truncate('s3fs_file_temp')
      ->execute();
  }

  // Create temporary table for folders which will allow for duplicates.
  // Folders will be written at the same time as the file data is written,
  // then will be merged with the files at the end.
  try {
    $folder_schema = $schema['s3fs_file'];
    unset($folder_schema['primary key'], $folder_schema['indexes']);
    db_create_table('s3fs_folder_temp', $folder_schema);
    $options = Database::getConnectionInfo('default');
    switch ($options['default']['driver']) {
      case 'pgsql':
        break;
      case 'sqlite':
        break;
      case 'mysql':
        db_query('ALTER TABLE {s3fs_folder_temp} CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin');
        break;
    }
  } catch (DatabaseSchemaObjectExistsException $e) {
    db_truncate('s3fs_folder_temp')
      ->execute();
  }

  // The $folders array is an associative array keyed by folder paths, which
  // is constructed as each filename is written to the DB. After all the files
  // are written, the folder paths are converted to metadata and written.
  $folders = array();
  $file_metadata_list = array();

  // Start by gathering all the existing folders. If we didn't do this, empty
  // folders would be lost, because they'd have no files from which to rebuild
  // themselves.
  $existing_folders = db_select('s3fs_file', 's')
    ->fields('s', array(
    'uri',
  ))
    ->condition('dir', 1, '=');
  $folder_counter = 0;
  foreach ($existing_folders
    ->execute()
    ->fetchCol(0) as $folder_uri) {
    $folders[rtrim($folder_uri, '/')] = TRUE;
    if ($folder_counter++ % 1000 == 0) {
      _s3fs_write_metadata($file_metadata_list, $folders);
    }
  }
  foreach ($iterator as $result) {
    if (!isset($result['Contents'])) {
      continue;
    }
    foreach ($result['Contents'] as $s3_metadata) {
      $key = $s3_metadata['Key'];

      // The root folder is an implementation detail that only appears on S3.
      // Files' URIs are not aware of it, so we need to remove it beforehand.
      if (!empty($config['root_folder'])) {
        $key = str_replace("{$config['root_folder']}/", '', $key);
      }

      // Figure out the scheme based on the key's folder prefix.
      $public_folder_name = !empty($config['public_folder']) ? $config['public_folder'] : 's3fs-public';
      $private_folder_name = !empty($config['private_folder']) ? $config['private_folder'] : 's3fs-private';
      if (strpos($key, "{$public_folder_name}/") === 0) {

        // Much like the root folder, the public folder name must be removed from URIs.
        $key = str_replace("{$public_folder_name}/", '', $key);
        $uri = "public://{$key}";
      }
      elseif (strpos($key, "{$private_folder_name}/") === 0) {
        $key = str_replace("{$private_folder_name}/", '', $key);
        $uri = "private://{$key}";
      }
      else {

        // No special prefix means it's an s3:// file.
        $uri = "s3://{$key}";
      }
      $max_uri_length = $schema['s3fs_file']['fields']['uri']['length'];
      if (strlen($uri) >= $max_uri_length) {
        watchdog('s3fs', 'URI "@uri" is too long, ignoring', array(
          '@uri' => $uri,
        ), WATCHDOG_WARNING);
        continue;
      }
      if ($uri[strlen($uri) - 1] == '/') {

        // Treat objects in S3 whose filenames end in a '/' as folders.
        // But don't store the '/' itself as part of the folder's uri.
        $folders[rtrim($uri, '/')] = TRUE;
      }
      else {

        // Only store the metadata for the latest version of the file.
        if (isset($s3_metadata['IsLatest']) && !$s3_metadata['IsLatest']) {
          continue;
        }

        // Files with no StorageClass are actually from the DeleteMarkers list,
        // rather then the Versions list. They represent a file which has been
        // deleted, so don't cache them.
        if (!isset($s3_metadata['StorageClass'])) {
          continue;
        }

        // Buckets with Versioning disabled set all files' VersionIds to "null".
        // If we see that, unset VersionId to prevent "null" from being written
        // to the DB.
        if (isset($s3_metadata['VersionId']) && $s3_metadata['VersionId'] == 'null') {
          unset($s3_metadata['VersionId']);
        }
        $file_metadata_list[] = _s3fs_convert_metadata($uri, $s3_metadata);
      }
      _s3fs_write_metadata($file_metadata_list, $folders);
    }
  }

  // Write folders.
  $query = db_select('s3fs_folder_temp')
    ->distinct();
  $query
    ->fields('s3fs_folder_temp');
  $folder_counter = 0;
  $result = $query
    ->execute();
  $insert_query = db_insert('s3fs_file_temp')
    ->fields(array(
    'uri',
    'filesize',
    'timestamp',
    'dir',
    'version',
  ));
  foreach ($result as $record) {
    $insert_query
      ->values((array) $record);

    // Flush every 1000 records.
    if ($folder_counter++ % 1000 == 0) {
      $insert_query
        ->execute();
    }
  }

  // Write any remainders.
  $insert_query
    ->execute();
  if (function_exists('drush_log')) {
    drush_log(dt('Flushed @folders folders to the file table.', array(
      '@folders' => $folder_counter,
    )));
  }

  // Cleanup.
  db_drop_table('s3fs_folder_temp');

  // Swap the temp table with the real table.
  db_rename_table('s3fs_file', 's3fs_file_old');
  db_rename_table('s3fs_file_temp', 's3fs_file');
  db_drop_table('s3fs_file_old');
  if (function_exists('drush_log')) {
    drush_log(dt('S3 File System cache refreshed.'));
  }
  else {
    drupal_set_message(t('S3 File System cache refreshed.'));
  }
}