webform_import.module in Webform Import 6
Same filename and directory in other branches
Allows delimited data files to be imported as results into webforms.
This is usefull for importing results from other systems in to Webform.
@author John C Jemmett <jjemmett@northwind-inc.com> @author Greg Bosen <gbosen@northwind-inc.com>
File
webform_import.moduleView source
<?php
/**
* @file
* Allows delimited data files to be imported as results into webforms.
*
* This is usefull for importing results from other systems in to Webform.
*
* @author John C Jemmett <jjemmett@northwind-inc.com>
* @author Greg Bosen <gbosen@northwind-inc.com>
*
*/
/**
* Implements hook_help().
*/
function webform_import_help($section = 'admin/help#webform', $arg = NULL) {
$output = '';
switch ($section) {
case 'node/%/webform/import':
$output = '<p>' . t('Webform import enables the upload of delimited files to fill in data.') . '</p>';
$output .= '<p>' . t('To upload data to this form:');
$items = array(
array(
'data' => 'Create a webform',
),
array(
'data' => 'Create a delimited file with the data you want uploaded',
'children' => array(
'data' => 'Make sure that you have a single header row consisting of either the Component names or keys',
),
),
array(
'data' => 'Go to node/??? > Results > Upload',
),
array(
'data' => 'Choose the type of delimiter, which type of header you are using and upload the file',
),
);
$output .= theme("item_list", $items, $title, "ol");
$output .= '</p>';
break;
}
return $output;
}
/**
* Implements hook_menu().
*/
function webform_import_menu() {
$items = array();
$items['node/%webform_menu/webform-results/upload'] = array(
'title' => 'Upload',
'page callback' => 'drupal_get_form',
'page arguments' => array(
'webform_import_form',
1,
),
'access callback' => 'node_access',
'access arguments' => array(
'update',
1,
),
'weight' => 10,
'type' => MENU_LOCAL_TASK,
);
$items['node/%webform_menu/webform-results/upload/%'] = array(
'title' => 'Get Template',
'page callback' => 'webform_import_csvtemplate',
'page arguments' => array(
1,
4,
),
'access callback' => 'node_access',
'access arguments' => array(
'update',
1,
),
'type' => MENU_CALLBACK,
);
$items['admin/settings/webform-import'] = array(
'title' => 'Webform Import',
'description' => 'Configure the settings used for webform import.',
'page callback' => 'drupal_get_form',
'page arguments' => array(
'webform_import_admin_settings',
),
'access arguments' => array(
'administer site configuration',
),
'type' => MENU_NORMAL_ITEM,
);
return $items;
}
/**
* Implements hook_admin_settings().
*
* Provide administrative options to configure webform imports functionality.
*/
function webform_import_admin_settings() {
$form = array();
$form['webform_import_queue_import'] = array(
'#type' => 'checkbox',
'#title' => t('Process imports via Queue'),
'#description' => t('Enable processing imports via Drupal Queue.'),
'#default_value' => variable_get('webform_import_queue_import', FALSE),
'#disabled' => !module_exists('drupal_queue'),
);
$form['webform_import_num_per_queue'] = array(
'#type' => 'select',
'#title' => t('Lines per Queue entry'),
'#description' => t('The number of lines to process per entry in the Drupal Queue.'),
'#options' => array(
100 => 100,
250 => 250,
500 => 500,
1000 => 1000,
2500 => 2500,
10000 => 10000,
),
'#default_value' => variable_get('webform_import_num_per_queue', 100),
'#disabled' => !module_exists('drupal_queue'),
);
$form['webform_import_file_destination'] = array(
'#type' => 'textfield',
'#title' => t('Upload destination'),
'#default_value' => variable_get('webform_import_file_destination', FALSE),
'#description' => t('The destination for webform import uploads.'),
);
return system_settings_form($form);
}
/**
* Creates a downloadable CSV template file corresponding to a Webform structure.
*
* @param $node
* The current webform node.
* @param $type
* The type of delimited file template to download.
*/
function webform_import_csvtemplate($node, $type) {
$types = _webform_import_field_key_options();
$filename = check_plain($node->title) . '_upload.csv';
$headers = array();
$node->webform['components']['-1'] = array(
'name' => 'User ID',
'form_key' => 'UID',
);
$node->webform['components']['-2'] = array(
'name' => 'Submission ID',
'form_key' => 'SID',
);
if (array_key_exists($type, $types)) {
foreach ($node->webform['components'] as $cid => $component) {
if ($component['type'] != 'file') {
$headers[] = $component[$type];
}
}
$csv = join(', ', $headers);
drupal_set_header('Content-type: application/octet-stream; charset=utf-8');
drupal_set_header('Content-disposition: attachment; filename=' . $filename);
print $csv;
}
else {
drupal_set_message(t('Invalid header type.'), 'warning');
$path = explode('/', $_GET['q']);
array_pop($path);
$path = join('/', $path);
drupal_goto(drupal_get_path_alias($path));
}
}
/**
* Client form generation function. Allows the user to upload a delimited file.
*
* @param $form_state
* The current form values of a submission, used in multipage webforms.
* @param $node
* The current webform node.
*
* @see webform_import_form_submit()
* @ingroup forms
*/
function webform_import_form(&$form_state, $node) {
global $user;
// Add a css class for all client forms.
$form['#attributes'] = array(
'class' => 'webform-import-form',
);
// Set the encoding type (necessary for file uploads).
$form['#attributes']['enctype'] = 'multipart/form-data';
$form['#redirect'] = 'node/' . $node->nid . '/webform-results/table';
$form['#submit'][] = 'webform_import_form_submit';
$form['details']['nid'] = array(
'#type' => 'value',
'#value' => $node->nid,
);
// @TODO Move these inline style elements into a css file.
$component_table_header = array(
array(
'data' => "<div style='position:relative'><span style='position:absolute;left:0'>" . t('Field Names') . "</span> <span style='position:absolute;right:0'>[ " . l('download template', 'node/' . $node->nid . '/webform-results/upload/name') . ' ]</span></div>',
),
array(
'data' => "<div style='position:relative'><span style='position:absolute;left:0'>" . t('Field Form Keys') . "</span> <span style='position:absolute;right:0'>[ " . l('download template', 'node/' . $node->nid . '/webform-results/upload/form_key') . ' ]</span></div>',
),
);
$component_table_rows = array();
$component_table_rows[] = array(
'Submission ID',
'SID',
);
$component_table_rows[] = array(
'User ID',
'UID',
);
foreach ($node->webform['components'] as $cid => $component) {
if ($component['type'] == 'file') {
continue;
}
$style = '';
if ($component['mandatory']) {
$style = 'font-weight: bold';
}
$component_table_rows[] = array(
array(
'data' => $component['name'],
'style' => $style,
),
array(
'data' => $component['form_key'],
'style' => $style,
),
);
}
$form['header'] = array(
'#value' => '<h2>' . t('Webform Import for "@title"', array(
'@title' => $node->title,
)) . '</h2>',
);
$form['instructions'] = array(
'#type' => 'fieldset',
'#title' => t('Instructions'),
'#collapsible' => TRUE,
'#collapsed' => TRUE,
);
$instructions_items = array(
array(
'data' => 'Component specific help:',
'children' => array(
'<strong>Date</strong>: must be in a format parsable by the php function <code>strtotime()</code> Any time data will be discarded.',
'<strong>Grid</strong>: option keys must be separated by commas and in the order of the questions in the webform. (e.g., <em>"red,male,car"</em> for the questions <em>"Favorite color, Gender, Type of automobile you drive"</em> respectively.)',
'<strong>File</strong>: * currently unable to handle this component.',
'<strong>Select</strong>: for multiselect answer keys must be separated by commas. (e.g., <em>"1,2,3"</em>)',
'<strong>Time</strong>: must be in a format parsable by the php function <code>strtotime()</code> Any date data will be discarded.',
),
),
);
$form['instructions']['instructions'] = array(
'#value' => '<p>' . t('Webform import enables the upload of delimited files to fill in data.') . '</p>' . '<p>' . t('The file must use the following values for column headers, anything else will be ignored.') . theme('table', $component_table_header, $component_table_rows) . '<p><small>' . t('* bold names are mandatory and must contain a value') . '</small><br/>' . '' . t('All rows with a submission id (SID) will be updated. Those without a submission id will be inserted.') . '<br/>' . '' . t('All rows with a user id (UID) will be associated with that user. Those without a user id will use the default setting.') . '</p><br/>' . theme("item_list", $instructions_items, NULL, "ol"),
);
$form['upload'] = array(
'#type' => 'file',
'#title' => t('Delimited data file'),
'#description' => t('Choose the data file containing the data you want uploaded. All rows with a submission id (SID) will be updated. Those without a submission id will be inserted.'),
);
$form['delimiter'] = array(
'#type' => 'select',
'#title' => t('File Delimiter'),
'#default_value' => '\\t',
'#options' => _webform_import_delimiter_options(),
'#description' => t('Delimiter for file being uploaded.'),
);
$users_list = array();
$result = db_query('SELECT uid, name FROM {users}');
while ($row = db_fetch_object($result)) {
if ($row->uid == 0) {
$row->name = "Anonymous";
}
$users_list[$row->uid] = $row->name;
}
$form['default_user'] = array(
'#type' => 'select',
'#title' => t('Default User'),
'#default_value' => $user->uid,
'#options' => $users_list,
'#description' => t('Select the default user to associate the submissions with (unless specified in the file).'),
);
$form['field_keys'] = array(
'#type' => 'select',
'#title' => t('Column header contains'),
'#default_value' => 'name',
'#options' => _webform_import_field_key_options(),
'#description' => t('What to use for the column key. (You probably want to choose form key if you have multiple fields with the same name.)'),
);
$form['buttons'] = array(
'#tree' => FALSE,
'#weight' => 1000,
);
// Add the submit button.
$form['buttons']['submit'] = array(
'#type' => 'submit',
'#value' => empty($node->webform['submit_text']) ? t('Submit') : $node->webform['submit_text'],
'#weight' => 10,
'#attributes' => variable_get('webform_import_queue_import', FALSE) ? array(
'onclick' => "this.disabled=true;this.value='Queueing...';this.form.submit();",
) : NULL,
);
return $form;
}
/**
* Form submission function to parse the delimited file and add to the database.
*
* @param $form
* The current form.
* @param $form_state
* The current form values of a submission.
*
* @see webform_import_form()
*/
function webform_import_form_submit($form, &$form_state) {
// Define your limits for the submission here.
$limits = array(
'extensions' => 'csv tsv txt',
);
$validators = array(
'file_validate_extensions' => array(
$limits['extensions'],
),
);
$webform = node_load(intval($form_state['values']['nid']));
$delimiter = $form_state['values']['delimiter'];
$delimiter = $delimiter == '\\t' ? '\\t' : $delimiter;
$field_key = $form_state['values']['field_keys'];
$default_user = $form_state['values']['default_user'];
$keys = array();
$keys[$field_key === 'name' ? 'Submission ID' : 'SID'] = array(
'form_key' => 'SID',
);
$keys[$field_key === 'name' ? 'User ID' : 'UID'] = array(
'form_key' => 'UID',
);
foreach ($webform->webform['components'] as $cid => $component) {
$component['cid'] = $cid;
$keys[trim($component[$field_key])] = $component;
}
if ($file = file_save_upload('upload', $validators, variable_get('webform_import_file_destination', FALSE))) {
// This makes php auto-detect mac line endings.
ini_set('auto_detect_line_endings', TRUE);
if (($handle = fopen($file->destination, 'r')) !== FALSE) {
// Get the column headings
$data = fgetcsv($handle, 0, $delimiter);
foreach ($data as $k => &$v) {
$v = _webform_import_csvfieldtrim($v);
if (!isset($keys[$v])) {
drupal_set_message(t('Can not find column @k in components list, skipping.', array(
'@k' => $v,
)), 'warning');
}
elseif (isset($keys[$v]['cid'])) {
$cids[$k] = $keys[$v]['cid'];
}
}
$fields = array_flip($data);
foreach ($keys as $k => $component) {
if (!empty($component['mandatory']) && $component['type'] != 'fieldset' && !isset($fields[$k])) {
form_set_error('upload', t('Column @k is required but could not be found in this file. Alter the file or the webform and try again.', array(
'@k' => $k,
)));
return;
}
}
$fields = $data;
$item = (object) array(
'filename' => $file->destination,
'start' => 1,
'limit' => FALSE,
'fields' => $fields,
'keys' => $keys,
'cids' => $cids,
'delimiter' => $delimiter,
'webform_nid' => $webform->nid,
'field_key' => $field_key,
'default_user' => $default_user,
'submission_time' => time(),
'remote_addr' => ip_address(),
);
if (variable_get('webform_import_queue_import', FALSE)) {
if (($handle = fopen($file->destination, 'r')) !== FALSE) {
$linecount = 0;
while (fgetcsv($handle, 0, $delimiter) !== FALSE) {
$linecount++;
}
fclose($handle);
}
$item->limit = variable_get('webform_import_num_per_queue', 100);
while ($item->start <= $linecount) {
webform_import_queue_create_item($item);
$item->start += variable_get('webform_import_num_per_queue', 100);
}
drupal_set_message(t('Upload file contained @num_lines lines that will be run @num at a time.', array(
'@num_lines' => $linecount,
'@num' => variable_get('webform_import_num_per_queue', 100),
)));
}
else {
webform_import_process($item);
}
}
else {
form_set_error('upload', t('File could not be opened for reading.'));
}
}
}
function webform_import_queue_create_item($item) {
// Create the queue item.
$queue = DrupalQueue::get('webform_import_queue_import');
return $queue
->createItem($item);
}
/**
* Parse the delimited file and add to the database.
*
* @param array $item
* All variables we need for the import process.
*/
function webform_import_process($item) {
module_load_include('inc', 'webform', 'includes/webform.submissions');
$webform = node_load(intval($item->webform_nid));
$field_key = $item->field_key;
$default_user = $item->default_user;
$delimiter = $item->delimiter;
$filename = $item->filename;
$fields = $item->fields;
$keys = $item->keys;
$cids = $item->cids;
$sids = array();
if (!empty($filename)) {
// This makes php auto-detect mac line endings.
ini_set('auto_detect_line_endings', TRUE);
if ($file = new SplFileObject($filename, 'r')) {
// This conditional is required because there is a bug in SplFileObject
// where seek(n) followed by fgetcsv will return the wrong line. In
// addition, this bug only occurs when n > 0 so n == 0 must be handled
// separately. See https://bugs.php.net/bug.php?id=46569.
if ($item->start == 1) {
$file
->seek(0);
$file
->fgets();
}
elseif ($item->start > 1) {
$file
->seek($item->start - 1);
}
$current_line = $item->start;
while (($item->limit == FALSE || $current_line < $item->start + $item->limit) && ($data = $file
->fgetcsv($delimiter))) {
$current_line++;
// If handle gets in a bad state this will keep an infinite loop from
// occuring and clean up after ourselves.
if ($data === FALSE) {
webform_import_log('Data was FALSE.', array(), 'error');
return;
}
// Ignore empty rows.
if ($data[0] === NULL) {
continue;
}
$sid = NULL;
$uid = NULL;
if (count($data) == count($fields)) {
$sub_array = array();
foreach ($data as $k => &$v) {
$component = $keys[$fields[$k]];
$v = _webform_import_csvfieldtrim($v);
if ($v == '') {
// Checking mandatory field for a value.
if ($component['mandatory']) {
webform_import_log('Required field has no value at row,col: @r,@c. Skipping this row!', array(
'@r' => $current_line,
'@c' => $k,
), 'warning');
continue 2;
}
else {
continue 1;
// Skip field if empty.
}
}
// Checking SID input security.
if ($component['form_key'] === 'SID') {
if (!is_numeric($v)) {
webform_import_log('Invalid Submission ID at row,col: @r,@c. Skipping this row!', array(
'@r' => $current_line,
'@c' => $k,
), 'warning');
continue 2;
}
else {
$sid = intval($v);
// Valid input security (integer values only).
}
}
elseif ($component['form_key'] === 'UID') {
if (!is_numeric($v)) {
webform_import_log('Invalid User ID at row,col: @r,@c. Skipping this row!', array(
'@r' => $current_line,
'@c' => $k,
), 'warning');
continue 2;
}
else {
$uid = intval($v);
// Valid input security (integer values only).
}
}
elseif (($cid = isset($cids[$k]) ? $cids[$k] : FALSE) !== FALSE) {
$type = $component['type'];
// Date and time components.
if ($type === 'date' || $type === 'time') {
if ($time = strtotime($v)) {
$v = $type === 'date' ? date('Y-m-d', $time) : date('H:i:s', $time);
}
else {
webform_import_log('Invalid datetime value at row,col: @r,@c. Skipping this row!', array(
'@r' => $current_line,
'@c' => $k,
), 'warning');
continue 2;
}
}
elseif ($type === 'grid') {
// Get the value for the grid components.
webform_component_include('select');
$options = array_flip(_webform_select_options_from_text($component['extra']['options'], TRUE));
$v = explode(',', $v);
foreach ($v as &$value) {
if (!empty($options[$value])) {
$value = $options[$value];
}
}
}
elseif ($type === 'select') {
// For select options, make sure we use the key and not the values.
webform_component_include('select');
$options = array_flip(_webform_select_options($component, TRUE));
// See if there are multiple values.
if ($component['extra']['multiple'] == 1) {
$v = explode(',', $v);
foreach ($v as &$value) {
if (!empty($options[$value])) {
$value = $options[$value];
}
}
}
else {
if (!empty($options[$v])) {
$v = $options[$v];
}
}
}
$sub_array[$cid] = $v;
}
}
$submission = (object) array(
'nid' => $webform->nid,
'uid' => $uid != NULL ? $uid : $default_user,
'submitted' => $item->submission_time,
'remote_addr' => $item->remote_addr,
'is_draft' => FALSE,
'data' => webform_submission_data($webform, $sub_array),
);
// Determine if INSERT or UPDATE based on inclusion of SID.
if ($sid != NULL) {
$submission->sid = $sid;
$sids[] = webform_submission_update($webform, $submission);
}
else {
$sids[] = webform_submission_insert($webform, $submission);
}
}
else {
webform_import_log('Row @c is malformed and will need to be fixed and resubmitted.', array(
'@c' => $current_line,
), 'warning');
}
}
if (!variable_get('webform_import_queue_import', FALSE)) {
if (!file_delete($filename)) {
webform_import_log('File could not be deleted (cleanup process). File: %file at path %path . !details', array(
'%file' => $file->filename,
'%path' => $file->destination,
'!results' => '<br />\\n<pre>' . htmlentities(print_r($form_state['values'], TRUE)) . '</pre>',
), 'error');
}
webform_import_log('Submission file uploaded to %title.', array(
'%title' => check_plain($webform->title),
));
webform_import_log('Uploaded @count webform submissions.', array(
'@count' => count($sids),
));
}
}
else {
webform_import_log('File read error. Could not read file %file at path %path.!details', array(
'%file' => $file->filename,
'%path' => $file->filepath,
'!results' => '<br />\\n<pre>' . htmlentities(print_r($form_state['values'], TRUE)) . '</pre>',
), 'error');
}
}
}
/**
* Log an entry regarding webform import and create a drupal message if appropriate.
*/
function webform_import_log($message, $substitutions = array(), $message_type = 'status') {
switch ($message_type) {
case 'warning':
$watchdog_level = WATCHDOG_WARNING;
break;
case 'error':
$watchdog_level = WATCHDOG_ERROR;
break;
case 'status':
default:
$watchdog_level = WATCHDOG_NOTICE;
break;
}
if (!variable_get('webform_import_queue_import', FALSE)) {
drupal_set_message(t($message, $substitutions), $message_type);
}
else {
watchdog('webform-import', $message, $substitutions, $watchdog_level);
}
}
/**
* Returns a list of value delimiters we can use.
*
* @return
* An array of key/value pairs for form options list.
*/
function _webform_import_delimiter_options() {
return array(
',' => t('Comma (,)'),
'\\t' => t('Tab (\\t)'),
';' => t('Semicolon (;)'),
':' => t('Colon (:)'),
'|' => t('Pipe (|)'),
'.' => t('Period (.)'),
' ' => t('Space ( )'),
);
}
/**
* Returns a list of field header types we can use.
*
* @return
* An array of key/value pairs for form options list.
*/
function _webform_import_field_key_options() {
return array(
'name' => t('Field Names'),
'form_key' => t('Field Form Keys'),
);
}
/**
* Returns a trimmed field value
*
* @param $value
* Field value to be trimmed.
* @return
* Trimmed field value.
*/
function _webform_import_csvfieldtrim($value) {
$value = trim($value);
// Strip off the beginning and ending quotes if necessary.
$value = preg_replace('/^".*"$/', '', $value);
// Remove control characters. Some editors add invalid EOL chars.
// fgetcsv does not handle unicode characters therefore we replace them
// manually. See http://bugs.php.net/bug.php?id=31632.
$value = str_replace('\\x00..\\x1F\\xfe\\xff', '', $value);
return $value;
}
/**
* Sets up the worker for the queue.
*
* @return string
*/
function webform_import_cron_queue_info() {
$queues = array();
if (variable_get('webform_import_queue_import', FALSE)) {
$queues['webform_import_queue_import'] = array(
'worker callback' => 'webform_import_process',
'time' => 90,
);
}
return $queues;
}Functions
Name |
Description |
|---|---|
| webform_import_admin_settings | Implements hook_admin_settings(). |
| webform_import_cron_queue_info | Sets up the worker for the queue. |
| webform_import_csvtemplate | Creates a downloadable CSV template file corresponding to a Webform structure. |
| webform_import_form | Client form generation function. Allows the user to upload a delimited file. |
| webform_import_form_submit | Form submission function to parse the delimited file and add to the database. |
| webform_import_help | Implements hook_help(). |
| webform_import_log | Log an entry regarding webform import and create a drupal message if appropriate. |
| webform_import_menu | Implements hook_menu(). |
| webform_import_process | Parse the delimited file and add to the database. |
| webform_import_queue_create_item | |
| _webform_import_csvfieldtrim | Returns a trimmed field value |
| _webform_import_delimiter_options | Returns a list of value delimiters we can use. |
| _webform_import_field_key_options | Returns a list of field header types we can use. |