migrate.module in Migrate 6.2
Same filename and directory in other branches
File
migrate.moduleView source
<?php
/**
* @file
* API and drush commands to support migration of data from external sources
* into a Drupal installation.
*/
require_once drupal_get_path('module', 'migrate') . '/includes/d7.inc';
// TODO:
// Continue hook_schema_alter() for map & message tables?
// Views hooks for map/message tables
// xlat support?
// Documentation
// Tests
define('MIGRATE_API_VERSION', 2);
/**
* Retrieve a list of all active migrations, ordered by dependencies. To be
* recognized, a class must be non-abstract, and derived from MigrationBase.
*
* @return
* Array of migration objects, keyed by the machine name.
*/
function migrate_migrations() {
static $migrations = array();
if (!empty($migrations)) {
return $migrations;
}
// Get list of modules implementing Migrate API - mainly, we're looking to
// make sure any dynamic migrations defined in hook_migrate_api() get registered.
migrate_get_module_apis(TRUE);
$dependent_migrations = array();
$required_migrations = array();
$result = db_select('migrate_status', 'ms')
->fields('ms', array(
'machine_name',
'class_name',
'arguments',
))
->execute();
foreach ($result as $row) {
if (class_exists($row->class_name)) {
$reflect = new ReflectionClass($row->class_name);
if (!$reflect
->isAbstract() && $reflect
->isSubclassOf('MigrationBase')) {
$arguments = unserialize($row->arguments);
if (!$arguments || !is_array($arguments)) {
$arguments = array();
}
$migration = MigrationBase::getInstance($row->machine_name, $row->class_name, $arguments);
$dependencies = $migration
->getDependencies();
if (count($dependencies) > 0) {
// Set classes with dependencies aside for reordering
$dependent_migrations[$row->machine_name] = $migration;
$required_migrations += $dependencies;
}
else {
// No dependencies, just add
$migrations[$row->machine_name] = $migration;
}
}
else {
MigrationBase::displayMessage(t('Class !class is no longer a valid concrete migration class', array(
'!class' => $row->class_name,
)));
}
}
else {
MigrationBase::displayMessage(t('Class !class no longer exists', array(
'!class' => $row->class_name,
)));
}
}
// Scan modules with dependencies - we'll take 20 passes at it before
// giving up
// TODO: Can we share code with _migrate_class_list()?
$iterations = 0;
while (count($dependent_migrations) > 0) {
if ($iterations++ > 20) {
$migration_names = implode(',', array_keys($dependent_migrations));
throw new MigrateException(t('Failure to sort migration list - most likely due ' . 'to circular dependencies involving !migration_names', array(
'!migration_names' => $migration_names,
)));
}
foreach ($dependent_migrations as $name => $migration) {
$ready = TRUE;
// Scan all the dependencies for this class and make sure they're all
// in the final list
foreach ($migration
->getDependencies() as $dependency) {
if (!isset($migrations[$dependency])) {
$ready = FALSE;
break;
}
}
if ($ready) {
// Yes they are! Move this class to the final list
$migrations[$name] = $migration;
unset($dependent_migrations[$name]);
}
}
}
// The migrations are now ordered according to their own dependencies - now order
// them by group
$groups = MigrateGroup::groups();
// Seed the final list by properly-ordered groups.
$final_migrations = array();
foreach ($groups as $name => $group) {
$final_migrations[$name] = array();
}
// Fill in the grouped list
foreach ($migrations as $machine_name => $migration) {
$final_migrations[$migration
->getGroup()
->getName()][$machine_name] = $migration;
}
// Then flatten the list
$migrations = array();
foreach ($final_migrations as $group_name => $group_migrations) {
foreach ($group_migrations as $machine_name => $migration) {
$migrations[$machine_name] = $migration;
}
}
return $migrations;
}
/**
* On request, scan the Drupal code registry for any new migration classes
* for us to register in migrate_status.
*/
function migrate_autoregister() {
// Make sure the registry is up-to-date on all available classes.
autoload_registry_update();
// Get list of modules implementing Migrate API
$modules = array_keys(migrate_get_module_apis(TRUE));
// Get list of classes we already know about
$existing_classes = array();
$result = db_query("SELECT class_name\n FROM {migrate_status}");
while ($row = db_fetch_object($result)) {
$existing_classes[$row->class_name] = $row->class_name;
}
// Discover class names registered with Drupal by modules implementing our API
$result = db_query("SELECT name\n FROM {autoload_registry}\n WHERE type='class' AND module IN (" . db_placeholders($modules, 'varchar') . ") AND filename NOT LIKE '%.test'", $modules);
while ($record = db_fetch_object($result)) {
$class_name = $record->name;
// If we already know about this class, skip it
if (isset($existing_classes[$class_name])) {
continue;
}
// Validate it's an implemented subclass of the parent class
// Ignore errors
try {
$class = new ReflectionClass($class_name);
} catch (Exception $e) {
continue;
}
if (!$class
->isAbstract() && $class
->isSubclassOf('MigrationBase')) {
// Verify that it's not a dynamic class (the implementor will be responsible
// for registering those).
$dynamic = call_user_func(array(
$class_name,
'isDynamic',
));
if (!$dynamic) {
// OK, this is a new non-dynamic migration class, register it
MigrationBase::registerMigration($class_name);
}
}
}
}
/**
* Invoke any available handlers attached to a given destination type.
* If any handlers have dependencies defined, they will be invoked after
* the specified handlers.
*
* @param $destination
* Destination type ('Node', 'User', etc.) - generally the same string as
* the destination class name without the MigrateDestination prefix.
* @param $method
* Method name such as 'prepare' (called at the beginning of an import operation)
* or 'complete' (called at the end of an import operation).
* @param ...
* Parameters to be passed to the handler.
*/
function migrate_handler_invoke_all($destination, $method) {
$args = func_get_args();
array_shift($args);
array_shift($args);
$return = array();
$class_list = _migrate_class_list('MigrateDestinationHandler');
$disabled = unserialize(variable_get('migrate_disabled_handlers', serialize(array())));
foreach ($class_list as $class_name => $handler) {
if (!in_array($class_name, $disabled) && $handler
->handlesType($destination) && method_exists($handler, $method)) {
migrate_instrument_start($class_name . '->' . $method);
$result = call_user_func_array(array(
$handler,
$method,
), $args);
migrate_instrument_stop($class_name . '->' . $method);
if (isset($result) && is_array($result)) {
$return = array_merge($return, $result);
}
elseif (isset($result)) {
$return[] = $result;
}
}
}
return $return;
}
/**
* Invoke any available handlers attached to a given field type.
* If any handlers have dependencies defined, they will be invoked after
* the specified handlers.
*
* @param $entity
* The object we are building up before calling example_save().
* @param $instance
* Array of info in the field instance, from field_info_instances().
* @param $values
* Array of incoming values, to be transformed into the appropriate structure
* for the field type.
* @param $method
* Handler method to call (defaults to prepare()).
*/
function migrate_field_handler_invoke_all($entity, array $instance, array $values, $method = 'prepare') {
$return = array();
$type = $instance['type'];
$class_list = _migrate_class_list('MigrateFieldHandler');
$disabled = unserialize(variable_get('migrate_disabled_handlers', serialize(array())));
foreach ($class_list as $class_name => $handler) {
if (!in_array($class_name, $disabled) && $handler
->handlesType($type) && method_exists($handler, $method)) {
migrate_instrument_start($class_name . '->' . $method);
$result = call_user_func_array(array(
$handler,
$method,
), array(
$entity,
$instance,
$values,
));
migrate_instrument_stop($class_name . '->' . $method);
if (isset($result) && is_array($result)) {
$return = array_merge($return, $result);
}
elseif (isset($result)) {
$return[] = $result;
}
}
}
return $return;
}
/**
* For a given parent class, identify and instantiate objects for any non-abstract
* classes derived from the parent, returning an array of the objects indexed by
* class name. The array will be ordered such that any classes with dependencies
* are listed after the classes they are dependent on.
*
* @param $parent_class
* Name of a class from which results will be derived.
* @return
* Array of objects, keyed by the class name.
*/
function _migrate_class_list($parent_class) {
// Get info on modules implementing Migrate API
static $module_info;
if (!isset($modules)) {
$module_info = migrate_get_module_apis();
}
$modules = array_keys($module_info);
static $class_lists = array();
if (!isset($class_lists[$parent_class])) {
$class_lists[$parent_class] = array();
if ($parent_class == 'MigrateDestinationHandler') {
$handler_key = 'destination handlers';
}
else {
$handler_key = 'field handlers';
}
// Add explicitly-registered handler classes
foreach ($module_info as $info) {
if (isset($info[$handler_key]) && is_array($info[$handler_key])) {
foreach ($info[$handler_key] as $handler_class) {
$class_lists[$parent_class][$handler_class] = new $handler_class();
}
}
}
// Avoid scrounging the registry for handler classes if possible.
if (variable_get('migrate_disable_autoregistration', FALSE)) {
return $class_lists[$parent_class];
}
$dependent_classes = array();
$required_classes = array();
// Discover class names registered with Drupal by modules implementing our API
$result = db_select('autoload_registry', 'r')
->fields('r', array(
'name',
))
->condition('type', 'class')
->condition('module', $modules, 'IN')
->condition('filename', '%.test', 'NOT LIKE')
->execute();
foreach ($result as $record) {
// Validate it's an implemented subclass of the parent class
// We can get funky errors here, ignore them (and the class that caused them)
try {
$class = new ReflectionClass($record->name);
} catch (Exception $e) {
continue;
}
if (!$class
->isAbstract() && $class
->isSubclassOf($parent_class)) {
// If the constructor has required parameters, this may fail. We will
// silently ignore - it is up to the implementor of such a class to
// instantiate it in hook_migrations_alter().
try {
$object = new $record->name();
} catch (Exception $e) {
unset($object);
}
if (isset($object)) {
$dependencies = $object
->getDependencies();
if (count($dependencies) > 0) {
// Set classes with dependencies aside for reordering
$dependent_classes[$record->name] = $object;
$required_classes += $dependencies;
}
else {
// No dependencies, just add
$class_lists[$parent_class][$record->name] = $object;
}
}
}
}
// Validate that each depended-on class at least exists
foreach ($required_classes as $class_name) {
if (!isset($dependent_classes[$class_name]) && !isset($class_lists[$parent_class][$class_name])) {
throw new MigrateException(t('Dependency on non-existent class !class - make sure ' . 'you have added the file defining !class to the .info file.', array(
'!class' => $class_name,
)));
}
}
// Scan modules with dependencies - we'll take 20 passes at it before
// giving up
$iterations = 0;
while (count($dependent_classes) > 0) {
if ($iterations++ > 20) {
$class_names = implode(',', array_keys($dependent_classes));
throw new MigrateException(t('Failure to sort class list - most likely due ' . 'to circular dependencies involving !class_names.', array(
'!class_names' => $class_names,
)));
}
foreach ($dependent_classes as $name => $object) {
$ready = TRUE;
// Scan all the dependencies for this class and make sure they're all
// in the final list
foreach ($object
->getDependencies() as $dependency) {
if (!isset($class_lists[$parent_class][$dependency])) {
$ready = FALSE;
break;
}
}
if ($ready) {
// Yes they are! Move this class to the final list
$class_lists[$parent_class][$name] = $object;
unset($dependent_classes[$name]);
}
}
}
}
return $class_lists[$parent_class];
}
/*
* Implementation of hook_migrate_api().
*/
function migrate_migrate_api() {
$api = array(
'api' => MIGRATE_API_VERSION,
'destination handlers' => array(
'MigrateTermNodeHandler',
'MigrateFieldsNodeHandler',
'MigratePathNodeHandler',
'MigrateProfileUserHandler',
'MigrateStatisticsEntityHandler',
),
'field handlers' => array(
'MigrateTextFieldHandler',
'MigrateValueFieldHandler',
'MigrateNodeReferenceFieldHandler',
'MigrateUserReferenceFieldHandler',
'MigrateFileFieldHandler',
'MigrateEmailFieldHandler',
),
);
return $api;
}
/**
* Get a list of modules that support the current migrate API.
*/
function migrate_get_module_apis($reset = FALSE) {
static $cache = NULL;
if ($reset) {
$cache = NULL;
}
if (!isset($cache)) {
$cache = array();
foreach (module_implements('migrate_api') as $module) {
$function = $module . '_migrate_api';
$info = $function();
if (isset($info['api']) && $info['api'] == MIGRATE_API_VERSION) {
$cache[$module] = $info;
// Register any migrations defined via the hook.
if (isset($info['migrations']) && is_array($info['migrations'])) {
foreach ($info['migrations'] as $machine_name => $arguments) {
MigrationBase::registerMigration($arguments['class_name'], $machine_name, $arguments);
}
}
}
else {
drupal_set_message(t('%function supports Migrate API version %modversion,
Migrate module API version is %version - migration support not loaded.', array(
'%function' => $function,
'%modversion' => $info['api'],
'%version' => MIGRATE_API_VERSION,
)));
}
}
}
return $cache;
}
/**
* Implementation of hook_watchdog().
* Find the migration that is currently running and notify it.
*
* @param array $log_entry
*/
function migrate_watchdog($log_entry) {
// Ensure that the Migration class exists, as different bootstrap phases may
// not have included migration.inc yet.
if (class_exists('Migration') && ($migration = Migration::currentMigration())) {
switch ($log_entry['severity']) {
case WATCHDOG_EMERG:
case WATCHDOG_ALERT:
case WATCHDOG_CRITICAL:
case WATCHDOG_ERROR:
$severity = MigrationBase::MESSAGE_ERROR;
break;
case WATCHDOG_WARNING:
$severity = MigrationBase::MESSAGE_WARNING;
break;
case WATCHDOG_NOTICE:
$severity = MigrationBase::MESSAGE_NOTICE;
break;
case WATCHDOG_DEBUG:
case WATCHDOG_INFO:
default:
$severity = MigrationBase::MESSAGE_INFORMATIONAL;
break;
}
$variables = is_array($log_entry['variables']) ? $log_entry['variables'] : array();
$migration
->saveMessage(t($log_entry['message'], $variables), $severity);
}
}
/**
* Resource functions modeled on Drupal's timer functions
*/
/**
* Implementation of hook_user()
*/
function migrate_user($op, &$edit, &$account, $category = NULL) {
switch ($op) {
case 'insert':
case 'update':
// Prevent from saving in users.data
if (isset($edit['migrate'])) {
$edit['migrate'] = NULL;
}
if (isset($edit['pathauto_perform_alias'])) {
// Prevent from saving in users.data
$edit['pathauto_perform_alias'] = NULL;
}
break;
}
}
/**
* Save memory usage with the specified name. If you start and stop the same
* memory name multiple times, the measured differences will be accumulated.
*
* @param name
* The name of the memory measurement.
*/
function migrate_memory_start($name) {
global $_migrate_memory;
$_migrate_memory[$name]['start'] = memory_get_usage();
$_migrate_memory[$name]['count'] = isset($_migrate_memory[$name]['count']) ? ++$_migrate_memory[$name]['count'] : 1;
}
/**
* Read the current memory value without recording the change.
*
* @param name
* The name of the memory measurement.
* @return
* The change in bytes since the last start.
*/
function migrate_memory_read($name) {
global $_migrate_memory;
if (isset($_migrate_memory[$name]['start'])) {
$stop = memory_get_usage();
$diff = $stop - $_migrate_memory[$name]['start'];
if (isset($_migrate_memory[$name]['bytes'])) {
$diff += $_migrate_memory[$name]['bytes'];
}
return $diff;
}
return $_migrate_memory[$name]['bytes'];
}
/**
* Stop the memory counter with the specified name.
*
* @param name
* The name of the memory measurement.
* @return
* A memory array. The array contains the number of times the memory has been
* started and stopped (count) and the accumulated memory difference value in bytes.
*/
function migrate_memory_stop($name) {
global $_migrate_memory;
if (isset($_migrate_memory[$name])) {
if (isset($_migrate_memory[$name]['start'])) {
$stop = memory_get_usage();
$diff = $stop - $_migrate_memory[$name]['start'];
if (isset($_migrate_memory[$name]['bytes'])) {
$_migrate_memory[$name]['bytes'] += $diff;
}
else {
$_migrate_memory[$name]['bytes'] = $diff;
}
unset($_migrate_memory[$name]['start']);
}
return $_migrate_memory[$name];
}
}
/**
* Start measuring time and (optionally) memory consumption over a section of code.
* Note that the memory consumption measurement is generally not useful in
* lower areas of the code, where data is being generated that will be freed
* by the next call to the same area. For example, measuring the memory
* consumption of db_query is not going to be helpful.
*
* @param $name
* The name of the measurement.
* @param $include_memory
* Measure both memory and timers. Defaults to FALSE (timers only).
*/
function migrate_instrument_start($name, $include_memory = FALSE) {
global $_migrate_track_memory, $_migrate_track_timer;
if ($_migrate_track_memory && $include_memory) {
migrate_memory_start($name);
}
if ($_migrate_track_timer) {
timer_start($name);
}
}
/**
* Stop measuring both memory and time consumption over a section of code.
*
* @param $name
* The name of the measurement.
*/
function migrate_instrument_stop($name) {
global $_migrate_track_memory, $_migrate_track_timer;
if ($_migrate_track_timer) {
timer_stop($name);
}
if ($_migrate_track_memory) {
migrate_memory_stop($name);
}
}
/**
* Call hook_migrate_overview for overall documentation on implemented migrations.
*/
function migrate_overview() {
$overview = '';
$results = module_invoke_all('migrate_overview');
foreach ($results as $result) {
$overview .= $result . ' ';
}
return $overview;
}
// TODO: The functions below are D6 functions of some potential use in D7, that
// haven't been updated/integrated yet
/**
* Implementation of hook_schema_alter().
*/
/*
function migrate_schema_alter(&$schema) {
// Check for table existence - at install time, hook_schema_alter() may be called
// before our install hook.
if (db_table_exists('migrate_content_sets')) {
$result = db_query("SELECT * FROM {migrate_content_sets}");
while ($content_set = db_fetch_object($result)) {
$maptablename = migrate_map_table_name($content_set->mcsid);
$msgtablename = migrate_message_table_name($content_set->mcsid);
// Get the proper field definition for the sourcekey
$view = views_get_view($content_set->view_name);
if (!$view) {
drupal_set_message(t('View !view does not exist - either (re)create this view, or
remove the migrate content set using it.', array('!view' => $content_set->view_name)));
continue;
}
// Must do this to load the database
$view->init_query();
// TODO: For now, PK must be in base_table
if (isset($view->base_database)) {
$tabledb = $view->base_database;
}
else {
$tabledb = 'default';
}
$tablename = $view->base_table;
$sourceschema = _migrate_inspect_schema($tablename, $tabledb);
// If the PK of the content set is defined, make sure we have a mapping table
$sourcekey = $content_set->sourcekey;
if ($sourcekey) {
$sourcefield = $sourceschema['fields'][$sourcekey];
if (!$sourcefield) {
// strip base table name if views prepended it
$baselen = drupal_strlen($tablename);
if (!strncasecmp($sourcekey, $tablename . '_', $baselen + 1)) {
$sourcekey = drupal_substr($sourcekey, $baselen + 1);
}
$sourcefield = $sourceschema['fields'][$sourcekey];
}
// We don't want serial fields to behave serially, so change to int
if ($sourcefield['type'] == 'serial') {
$sourcefield['type'] = 'int';
}
$schema[$maptablename] = _migrate_map_table_schema($sourcefield);
$schema[$maptablename]['name'] = $maptablename;
$schema[$msgtablename] = _migrate_message_table_schema($sourcefield);
$schema[$msgtablename]['name'] = $msgtablename;
}
}
}
}
*/
/*
* Translate URIs from an old site to the new one
* Requires adding RewriteRules to .htaccess. For example, if the URLs
* for news articles had the form
* http://example.com/issues/news/[OldID].html, use this rule:
*
* RewriteRule ^issues/news/([0-9]+).html$ /migrate/xlat/node/$1 [L]
*
* @param $contenttype
* Content type to translate (e.g., 'node', 'user', etc.)
* @param $oldid
* Primary key from input view
*/
function migrate_xlat($contenttype, $oldid) {
if ($contenttype && $oldid) {
$newid = _migrate_xlat_get_new_id($contenttype, $oldid);
if ($newid) {
$uri = migrate_invoke_all("xlat_{$contenttype}", $newid);
drupal_goto($uri[0], NULL, NULL, 301);
}
}
}
/*
* Helper function to translate an ID from a source file to the corresponding
* Drupal-side ID (nid, uid, etc.)
* Note that the result may be ambiguous - for example, if you are importing
* nodes from different content sets, they might have overlapping source IDs.
*
* @param $contenttype
* Content type to translate (e.g., 'node', 'user', etc.)
* @param $oldid
* Primary key from input view
* @return
* Drupal-side ID of the object
*/
function _migrate_xlat_get_new_id($contenttype, $oldid) {
$result = db_query("SELECT mcsid\n FROM {migrate_content_sets}\n WHERE contenttype='%s'", $contenttype);
while ($row = db_fetch_object($result)) {
static $maptables = array();
if (!isset($maptables[$row->mcsid])) {
$maptables[$row->mcsid] = migrate_map_table_name($row->mcsid);
}
$sql = "SELECT destid\n FROM {" . $maptables[$row->mcsid] . "}\n WHERE sourceid='%s'";
$id = db_result(db_query($sql, $oldid));
if ($id) {
return $id;
}
}
return NULL;
}
/**
* Print a SQL string from a DBTNG Query object. Includes quoted arguments.
* Modified slightly from devel module.
*
* @param $query
* A Query object.
* @param $return
* Whether to return or print the string. Default to FALSE.
* @param $name
* Optional name for identifying the output.
*/
function dpq($query, $return = FALSE, $name = NULL) {
$query
->preExecute();
$sql = (string) $query;
$quoted = array();
$connection = Database::getConnection();
foreach ((array) $query
->arguments() as $key => $val) {
$quoted[$key] = $connection
->quote($val);
}
$sql = strtr($sql, $quoted);
return $sql;
}
Functions
Name | Description |
---|---|
dpq | Print a SQL string from a DBTNG Query object. Includes quoted arguments. Modified slightly from devel module. |
migrate_autoregister | On request, scan the Drupal code registry for any new migration classes for us to register in migrate_status. |
migrate_field_handler_invoke_all | Invoke any available handlers attached to a given field type. If any handlers have dependencies defined, they will be invoked after the specified handlers. |
migrate_get_module_apis | Get a list of modules that support the current migrate API. |
migrate_handler_invoke_all | Invoke any available handlers attached to a given destination type. If any handlers have dependencies defined, they will be invoked after the specified handlers. |
migrate_instrument_start | Start measuring time and (optionally) memory consumption over a section of code. Note that the memory consumption measurement is generally not useful in lower areas of the code, where data is being generated that will be freed by the next call to the… |
migrate_instrument_stop | Stop measuring both memory and time consumption over a section of code. |
migrate_memory_read | Read the current memory value without recording the change. |
migrate_memory_start | Save memory usage with the specified name. If you start and stop the same memory name multiple times, the measured differences will be accumulated. |
migrate_memory_stop | Stop the memory counter with the specified name. |
migrate_migrate_api | |
migrate_migrations | Retrieve a list of all active migrations, ordered by dependencies. To be recognized, a class must be non-abstract, and derived from MigrationBase. |
migrate_overview | Call hook_migrate_overview for overall documentation on implemented migrations. |
migrate_user | Implementation of hook_user() |
migrate_watchdog | Implementation of hook_watchdog(). Find the migration that is currently running and notify it. |
migrate_xlat | |
_migrate_class_list | For a given parent class, identify and instantiate objects for any non-abstract classes derived from the parent, returning an array of the objects indexed by class name. The array will be ordered such that any classes with dependencies are listed… |
_migrate_xlat_get_new_id |
Constants
Name | Description |
---|---|
MIGRATE_API_VERSION |