View source  
  <?php
function robotstxt_help($path, $arg) {
  switch ($path) {
    case 'admin/help#robotstxt':
      return '<p>' . t('In a multisite environment, there is no mechanism for having a separate robots.txt file for each site. This module addresses that need by letting you administer the robots.txt file from the settings interface.') . '</p>';
    case 'admin/config/search/robotstxt':
      if (file_exists(DRUPAL_ROOT . '/robots.txt')) {
        drupal_set_message(t('One or more problems have been detected with the RobotsTxt configuration. Check the <a href="@status">status report</a> for more information.', array(
          '@status' => url('admin/reports/status'),
        )), 'warning');
      }
      
      return t('See <a href="http://www.robotstxt.org/">http://www.robotstxt.org/</a> for more information concerning how to write your <a href="@robotstxt">robots.txt</a> file.', array(
        '@robotstxt' => base_path() . 'robots.txt',
      ));
  }
}
function robotstxt_permission() {
  return array(
    'administer robots.txt' => array(
      'title' => t('Administer robots.txt'),
      'description' => t('Perform maintenance tasks for robots.txt.'),
    ),
  );
}
function robotstxt_menu() {
  $items['robots.txt'] = array(
    'page callback' => 'robotstxt_robots',
    'access callback' => TRUE,
    'type' => MENU_CALLBACK,
  );
  $items['admin/config/search/robotstxt'] = array(
    'title' => 'RobotsTxt',
    'description' => 'Manage your robots.txt file.',
    'page callback' => 'drupal_get_form',
    'page arguments' => array(
      'robotstxt_admin_settings',
    ),
    'access arguments' => array(
      'administer robots.txt',
    ),
    'file' => 'robotstxt.admin.inc',
  );
  return $items;
}
function robotstxt_robots() {
  $content = array();
  $content[] = variable_get('robotstxt', '');
  
  if ($additions = module_invoke_all('robotstxt')) {
    $content = array_merge($content, $additions);
  }
  
  $content = array_map('trim', $content);
  $content = array_filter($content);
  drupal_add_http_header('Content-type', 'text/plain');
  echo implode("\n", $content);
  drupal_page_footer();
  exit;
}