diff --git a/CsvImportPlugin.php b/CsvImportPlugin.php new file mode 100644 index 0000000..0ef398a --- /dev/null +++ b/CsvImportPlugin.php @@ -0,0 +1,245 @@ + +* plugins.CsvImport.columnDelimiter = "," +* plugins.CsvImport.memoryLimit = "128M" +* plugins.CsvImport.requiredExtension = "txt" +* plugins.CsvImport.requiredMimeType = "text/csv" +* plugins.CsvImport.maxFileSize = "10M" +* plugins.CsvImport.fileDestination = "/tmp" +* plugins.CsvImport.batchSize = "1000" +* +* +* All of the above settings are optional. If not given, CsvImport uses the +* following default values: +* +* memoryLimit = current script limit +* requiredExtension = "txt" or "csv" +* requiredMimeType = "text/csv" +* maxFileSize = current system upload limit +* fileDestination = current system temporary dir (via sys_get_temp_dir()) +* batchSize = 0 (no batching) +* +* +* Set a high memory limit to avoid memory allocation issues with imports. +* Examples include 128M, 1G, and -1. This will set PHP's memory_limit setting +* directly, see PHP's documentation for more info on formatting this number. +* Be advised that many web hosts set a maximum memory limit, so this setting +* may be ignored if it exceeds the maximum allowable limit. Check with your web +* host for more information. +* +* Note that 'maxFileSize' will not affect post_max_size or upload_max_filesize +* as is set in php.ini. Having a maxFileSize that exceeds either +* will still result in errors that prevent the file upload. +* +* batchSize: Setting for advanced users. If you find that your long-running +* imports are using too much memory or otherwise hogging system resources, +* set this value to split your import into multiple jobs based on the +* number of CSV rows to process per job. +* +* For example, if you have a CSV with 150000 rows, setting a batchSize +* of 5000 would cause the import to be split up over 30 separate jobs. +* Note that these jobs run sequentially based on the results of prior +* jobs, meaning that the import cannot be parallelized. The first job +* will import 5000 rows and then spawn the next job, and so on until +* the import is completed. +* +* +* @copyright Copyright 2008-2012 Roy Rosenzweig Center for History and New Media +* @license http://www.gnu.org/licenses/gpl-3.0.txt GNU GPLv3 +* @package CsvImport +*/ + +defined('CSV_IMPORT_DIRECTORY') or define('CSV_IMPORT_DIRECTORY', dirname(__FILE__)); + +/** + * Csv Import plugin. + */ +class CsvImportPlugin extends Omeka_Plugin_AbstractPlugin +{ + const MEMORY_LIMIT_OPTION_NAME = 'csv_import_memory_limit'; + const PHP_PATH_OPTION_NAME = 'csv_import_php_path'; + + /** + * @var array Hooks for the plugin. + */ + protected $_hooks = array( + 'install', + 'uninstall', + 'upgrade', + 'initialize', + 'admin_head', + 'define_acl', + ); + + /** + * @var array Filters for the plugin. + */ + protected $_filters = array( + 'admin_navigation_main', + ); + + /** + * @var array Options and their default values. + */ + protected $_options = array( + self::MEMORY_LIMIT_OPTION_NAME => '', + self::PHP_PATH_OPTION_NAME => '', + CsvImport_RowIterator::COLUMN_DELIMITER_OPTION_NAME => CsvImport_RowIterator::DEFAULT_COLUMN_DELIMITER, + CsvImport_ColumnMap_Element::ELEMENT_DELIMITER_OPTION_NAME => CsvImport_ColumnMap_Element::DEFAULT_ELEMENT_DELIMITER, + CsvImport_ColumnMap_Tag::TAG_DELIMITER_OPTION_NAME => CsvImport_ColumnMap_Tag::DEFAULT_TAG_DELIMITER, + CsvImport_ColumnMap_File::FILE_DELIMITER_OPTION_NAME => CsvImport_ColumnMap_File::DEFAULT_FILE_DELIMITER, + ); + + /** + * Install the plugin. + */ + public function hookInstall() + { + $db = $this->_db; + + // create csv imports table + $db->query("CREATE TABLE IF NOT EXISTS `{$db->prefix}csv_import_imports` ( + `id` int(10) unsigned NOT NULL auto_increment, + `item_type_id` int(10) unsigned NULL, + `collection_id` int(10) unsigned NULL, + `owner_id` int unsigned NOT NULL, + `delimiter` varchar(1) collate utf8_unicode_ci NOT NULL, + `original_filename` text collate utf8_unicode_ci NOT NULL, + `file_path` text collate utf8_unicode_ci NOT NULL, + `file_position` bigint unsigned NOT NULL, + `status` varchar(255) collate utf8_unicode_ci, + `skipped_row_count` int(10) unsigned NOT NULL, + `skipped_item_count` int(10) unsigned NOT NULL, + `is_public` tinyint(1) default '0', + `is_featured` tinyint(1) default '0', + `serialized_column_maps` text collate utf8_unicode_ci NOT NULL, + `added` timestamp NOT NULL default '2000-01-01 00:00:00', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;"); + + // create csv imported items table + $db->query("CREATE TABLE IF NOT EXISTS `{$db->prefix}csv_import_imported_items` ( + `id` int(10) unsigned NOT NULL auto_increment, + `item_id` int(10) unsigned NOT NULL, + `import_id` int(10) unsigned NOT NULL, + PRIMARY KEY (`id`), + KEY (`import_id`), + UNIQUE (`item_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;"); + + $this->_installOptions(); + } + + /** + * Uninstall the plugin. + */ + public function hookUninstall() + { + $db = $this->_db; + + // drop the tables + $sql = "DROP TABLE IF EXISTS `{$db->prefix}csv_import_imports`"; + $db->query($sql); + $sql = "DROP TABLE IF EXISTS `{$db->prefix}csv_import_imported_items`"; + $db->query($sql); + + $this->_uninstallOptions(); + } + + /** + * Upgrade the plugin. + */ + public function hookUpgrade($args) + { + $oldVersion = $args['old_version']; + $newVersion = $args['new_version']; + $db = $this->_db; + + // Do this first because MySQL will complain about any ALTERs to a table with an + // invalid default if we don't fix it first + if (version_compare($oldVersion, '2.0.3', '<=')) { + $sql = "ALTER TABLE `{$db->prefix}csv_import_imports` MODIFY `added` timestamp NOT NULL default '2000-01-01 00:00:00'"; + $db->query($sql); + } + + if (version_compare($oldVersion, '2.0-dev', '<=')) { + $sql = "UPDATE `{$db->prefix}csv_import_imports` SET `status` = ? WHERE `status` = ?"; + $db->query($sql, array('other_error', 'error')); + } + + if (version_compare($oldVersion, '2.0', '<=')) { + set_option(CsvImport_RowIterator::COLUMN_DELIMITER_OPTION_NAME, CsvImport_RowIterator::DEFAULT_COLUMN_DELIMITER); + set_option(CsvImport_ColumnMap_Element::ELEMENT_DELIMITER_OPTION_NAME, CsvImport_ColumnMap_Element::DEFAULT_ELEMENT_DELIMITER); + set_option(CsvImport_ColumnMap_Tag::TAG_DELIMITER_OPTION_NAME, CsvImport_ColumnMap_Tag::DEFAULT_TAG_DELIMITER); + set_option(CsvImport_ColumnMap_File::FILE_DELIMITER_OPTION_NAME, CsvImport_ColumnMap_File::DEFAULT_FILE_DELIMITER); + } + + if(version_compare($oldVersion, '2.0.1', '<=')) { + $sql = "ALTER TABLE `{$db->prefix}csv_import_imports` CHANGE `item_type_id` `item_type_id` INT( 10 ) UNSIGNED NULL , + CHANGE `collection_id` `collection_id` INT( 10 ) UNSIGNED NULL + "; + $db->query($sql); + } + } + + /** + * Add the translations. + */ + public function hookInitialize() + { + add_translation_source(dirname(__FILE__) . '/languages'); + } + + /** + * Define the ACL. + * + * @param array $args + */ + public function hookDefineAcl($args) + { + $acl = $args['acl']; // get the Zend_Acl + + $acl->addResource('CsvImport_Index'); + + // Hack to disable CRUD actions. + $acl->deny(null, 'CsvImport_Index', array('show', 'add', 'edit', 'delete')); + $acl->deny('admin', 'CsvImport_Index'); + } + + /** + * Configure admin theme header. + * + * @param array $args + */ + public function hookAdminHead($args) + { + $request = Zend_Controller_Front::getInstance()->getRequest(); + if ($request->getModuleName() == 'csv-import') { + queue_css_file('csv-import-main'); + queue_js_file('csv-import'); + } + } + + /** + * Add the Simple Pages link to the admin main navigation. + * + * @param array Navigation array. + * @return array Filtered navigation array. + */ + public function filterAdminNavigationMain($nav) + { + $nav[] = array( + 'label' => __('CSV Import'), + 'uri' => url('csv-import'), + 'resource' => 'CsvImport_Index', + 'privilege' => 'index', + ); + return $nav; + } +} diff --git a/controllers/IndexController.php b/controllers/IndexController.php new file mode 100644 index 0000000..f5be898 --- /dev/null +++ b/controllers/IndexController.php @@ -0,0 +1,392 @@ +session = new Zend_Session_Namespace('CsvImport'); + $this->_helper->db->setDefaultModelName('CsvImport_Import'); + } + + /** + * Configure a new import. + */ + public function indexAction() + { + $form = $this->_getMainForm(); + $this->view->form = $form; + + if (!$this->getRequest()->isPost()) { + return; + } + + if (!$form->isValid($this->getRequest()->getPost())) { + $this->_helper->flashMessenger(__('Invalid form input. Please see errors below and try again.'), 'error'); + return; + } + + if (!$form->csv_file->receive()) { + $this->_helper->flashMessenger(__('Error uploading file. Please try again.'), 'error'); + return; + } + + $filePath = $form->csv_file->getFileName(); + $columnDelimiter = $form->getValue('column_delimiter'); + + $file = new CsvImport_File($filePath, $columnDelimiter); + + if (!$file->parse()) { + $this->_helper->flashMessenger(__('Your file is incorrectly formatted.') + . ' ' . $file->getErrorString(), 'error'); + return; + } + + $this->session->setExpirationHops(2); + $this->session->originalFilename = $_FILES['csv_file']['name']; + $this->session->filePath = $filePath; + + $this->session->columnDelimiter = $columnDelimiter; + $this->session->columnNames = $file->getColumnNames(); + $this->session->columnExamples = $file->getColumnExamples(); + + $this->session->fileDelimiter = $form->getValue('file_delimiter'); + $this->session->tagDelimiter = $form->getValue('tag_delimiter'); + $this->session->elementDelimiter = $form->getValue('element_delimiter'); + $this->session->itemTypeId = $form->getValue('item_type_id'); + $this->session->itemsArePublic = $form->getValue('items_are_public'); + $this->session->itemsAreFeatured = $form->getValue('items_are_featured'); + $this->session->collectionId = $form->getValue('collection_id'); + + $this->session->automapColumnNamesToElements = $form->getValue('automap_columns_names_to_elements'); + + $this->session->ownerId = $this->getInvokeArg('bootstrap')->currentuser->id; + + // All is valid, so we save settings. + set_option(CsvImport_RowIterator::COLUMN_DELIMITER_OPTION_NAME, $this->session->columnDelimiter); + set_option(CsvImport_ColumnMap_Element::ELEMENT_DELIMITER_OPTION_NAME, $this->session->elementDelimiter); + set_option(CsvImport_ColumnMap_Tag::TAG_DELIMITER_OPTION_NAME, $this->session->tagDelimiter); + set_option(CsvImport_ColumnMap_File::FILE_DELIMITER_OPTION_NAME, $this->session->fileDelimiter); + + if ($form->getValue('omeka_csv_export')) { + $this->_helper->redirector->goto('check-omeka-csv'); + } + + $this->_helper->redirector->goto('map-columns'); + } + + /** + * Map the columns for an import. + */ + public function mapColumnsAction() + { + if (!$this->_sessionIsValid()) { + $this->_helper->flashMessenger(__('Import settings expired. Please try again.'), 'error'); + $this->_helper->redirector->goto('index'); + return; + } + + require_once CSV_IMPORT_DIRECTORY . '/forms/Mapping.php'; + $form = new CsvImport_Form_Mapping(array( + 'itemTypeId' => $this->session->itemTypeId, + 'columnNames' => $this->session->columnNames, + 'columnExamples' => $this->session->columnExamples, + 'fileDelimiter' => $this->session->fileDelimiter, + 'tagDelimiter' => $this->session->tagDelimiter, + 'elementDelimiter' => $this->session->elementDelimiter, + 'automapColumnNamesToElements' => $this->session->automapColumnNamesToElements + )); + $this->view->form = $form; + + if (!$this->getRequest()->isPost()) { + return; + } + if (!$form->isValid($this->getRequest()->getPost())) { + $this->_helper->flashMessenger(__('Invalid form input. Please try again.'), 'error'); + return; + } + + $columnMaps = $form->getColumnMaps(); + if (count($columnMaps) == 0) { + $this->_helper->flashMessenger(__('Please map at least one column to an element, file, or tag.'), 'error'); + return; + } + + $csvImport = new CsvImport_Import(); + foreach ($this->session->getIterator() as $key => $value) { + $setMethod = 'set' . ucwords($key); + if (method_exists($csvImport, $setMethod)) { + $csvImport->$setMethod($value); + } + } + $csvImport->setColumnMaps($columnMaps); + if ($csvImport->queue()) { + $this->_dispatchImportTask($csvImport, CsvImport_ImportTask::METHOD_START); + $this->_helper->flashMessenger(__('Import started. Reload this page for status updates.'), 'success'); + } else { + $this->_helper->flashMessenger(__('Import could not be started. Please check error logs for more details.'), 'error'); + } + + $this->session->unsetAll(); + $this->_helper->redirector->goto('browse'); + } + + /** + * For import of Omeka.net CSV. + * Check if all needed Elements are present. + */ + public function checkOmekaCsvAction() + { + $elementTable = get_db()->getTable('Element'); + $skipColumns = array('itemType', + 'collection', + 'tags', + 'public', + 'featured', + 'file'); + + $skipColumnsWrapped = array(); + foreach($skipColumns as $skipColumn) { + $skipColumnsWrapped[] = "'" . $skipColumn . "'"; + } + $skipColumnsText = '( ' . implode(', ', $skipColumnsWrapped) . ' )'; + + if (empty($this->session->columnNames)) { + $this->_helper->redirector->goto('index'); + } + + $hasError = false; + foreach ($this->session->columnNames as $columnName){ + if (!in_array($columnName, $skipColumns)) { + $data = explode(':', $columnName); + if (count($data) != 2) { + $this->_helper->flashMessenger(__('Invalid column names. Column names must either be one of the following %s, or have the following format: {ElementSetName}:{ElementName}', $skipColumnsText), 'error'); + $hasError = true; + break; + } + } + } + + if (!$hasError) { + foreach ($this->session->columnNames as $columnName){ + if (!in_array($columnName, $skipColumns)) { + $data = explode(':', $columnName); + //$data is like array('Element Set Name', 'Element Name'); + $elementSetName = $data[0]; + $elementName = $data[1]; + $element = $elementTable->findByElementSetNameAndElementName($elementSetName, $elementName); + if (empty($element)) { + $this->_helper->flashMessenger(__('Element "%s" is not found in element set "%s"', array($elementName, $elementSetName)), 'error'); + $hasError = true; + } + } + } + } + + if (!$hasError) { + $this->_helper->redirector->goto('omeka-csv'); + } + } + + /** + * Create and queue a new import from Omeka.net. + */ + public function omekaCsvAction() + { + // specify the export format's file and tag delimiters + // do not allow the user to specify it + $fileDelimiter = ','; + $tagDelimiter = ','; + + $headings = $this->session->columnNames; + $columnMaps = array(); + foreach ($headings as $heading) { + switch ($heading) { + case 'collection': + $columnMaps[] = new CsvImport_ColumnMap_Collection($heading); + break; + case 'itemType': + $columnMaps[] = new CsvImport_ColumnMap_ItemType($heading); + break; + case 'file': + $columnMaps[] = new CsvImport_ColumnMap_File($heading, $fileDelimiter); + break; + case 'tags': + $columnMaps[] = new CsvImport_ColumnMap_Tag($heading, $tagDelimiter); + break; + case 'public': + $columnMaps[] = new CsvImport_ColumnMap_Public($heading); + break; + case 'featured': + $columnMaps[] = new CsvImport_ColumnMap_Featured($heading); + break; + default: + $columnMaps[] = new CsvImport_ColumnMap_ExportedElement($heading); + break; + } + } + $csvImport = new CsvImport_Import(); + + //this is the clever way that mapColumns action sets the values passed along from indexAction + //many will be irrelevant here, since CsvImport allows variable itemTypes and Collection + + //@TODO: check if variable itemTypes and Collections breaks undo. It probably should, actually + foreach ($this->session->getIterator() as $key => $value) { + $setMethod = 'set' . ucwords($key); + if (method_exists($csvImport, $setMethod)) { + $csvImport->$setMethod($value); + } + } + $csvImport->setColumnMaps($columnMaps); + if ($csvImport->queue()) { + $this->_dispatchImportTask($csvImport, CsvImport_ImportTask::METHOD_START); + $this->_helper->flashMessenger(__('Import started. Reload this page for status updates.'), 'success'); + } else { + $this->_helper->flashMessenger(__('Import could not be started. Please check error logs for more details.'), 'error'); + } + $this->session->unsetAll(); + $this->_helper->redirector->goto('browse'); + } + + /** + * Browse the imports. + */ + public function browseAction() + { + if (!$this->_getParam('sort_field')) { + $this->_setParam('sort_field', 'added'); + $this->_setParam('sort_dir', 'd'); + } + parent::browseAction(); + } + + /** + * Undo the import. + */ + public function undoImportAction() + { + $csvImport = $this->_helper->db->findById(); + if ($csvImport->queueUndo()) { + $this->_dispatchImportTask($csvImport, CsvImport_ImportTask::METHOD_UNDO); + $this->_helper->flashMessenger(__('Undo import started. Reload this page for status updates.'), 'success'); + } else { + $this->_helper->flashMessenger(__('Undo import could not be started. Please check error logs for more details.'), 'error'); + } + + $this->_helper->redirector->goto('browse'); + } + + /** + * Clear the import history. + */ + public function clearHistoryAction() + { + $csvImport = $this->_helper->db->findById(); + $importedItemCount = $csvImport->getImportedItemCount(); + + if ($csvImport->isUndone() || + $csvImport->isUndoImportError() || + $csvImport->isOtherError() || + ($csvImport->isImportError() && $importedItemCount == 0)) { + $csvImport->delete(); + $this->_helper->flashMessenger(__('Cleared import from the history.'), 'success'); + } else { + $this->_helper->flashMessenger(__('Cannot clear import history.'), 'error'); + } + $this->_helper->redirector->goto('browse'); + } + + /** + * Get the main Csv Import form. + * + * @return CsvImport_Form_Main + */ + protected function _getMainForm() + { + require_once CSV_IMPORT_DIRECTORY . '/forms/Main.php'; + $csvConfig = $this->_getPluginConfig(); + $form = new CsvImport_Form_Main($csvConfig); + return $form; + } + + /** + * Returns the plugin configuration + * + * @return array + */ + protected function _getPluginConfig() + { + if (!$this->_pluginConfig) { + $config = $this->getInvokeArg('bootstrap')->config->plugins; + if ($config && isset($config->CsvImport)) { + $this->_pluginConfig = $config->CsvImport->toArray(); + } + if (!array_key_exists('fileDestination', $this->_pluginConfig)) { + $this->_pluginConfig['fileDestination'] = + Zend_Registry::get('storage')->getTempDir(); + } + } + return $this->_pluginConfig; + } + + /** + * Returns whether the session is valid. + * + * @return boolean + */ + protected function _sessionIsValid() + { + $requiredKeys = array('itemsArePublic', + 'itemsAreFeatured', + 'collectionId', + 'itemTypeId', + 'ownerId'); + foreach ($requiredKeys as $key) { + if (!isset($this->session->$key)) { + return false; + } + } + return true; + } + + /** + * Dispatch an import task. + * + * @param CsvImport_Import $csvImport The import object + * @param string $method The method name to run in the CsvImport_Import object + */ + protected function _dispatchImportTask($csvImport, $method = null) + { + if ($method === null) { + $method = CsvImport_ImportTask::METHOD_START; + } + $csvConfig = $this->_getPluginConfig(); + + $jobDispatcher = Zend_Registry::get('job_dispatcher'); + $jobDispatcher->setQueueName(CsvImport_ImportTask::QUEUE_NAME); + try { + $jobDispatcher->sendLongRunning('CsvImport_ImportTask', + array( + 'importId' => $csvImport->id, + 'memoryLimit' => @$csvConfig['memoryLimit'], + 'batchSize' => @$csvConfig['batchSize'], + 'method' => $method, + ) + ); + } catch (Exception $e) { + $csvImport->setStatus(CsvImport_Import::OTHER_ERROR); + throw $e; + } + } +} diff --git a/csv_files/test.csv b/csv_files/test.csv new file mode 100644 index 0000000..00539e9 --- /dev/null +++ b/csv_files/test.csv @@ -0,0 +1,4 @@ +title, creator, description, tags, file +"Walden", "Henry David Thoreau", "A man seeks simplicity.", "book, classic, New England", "http://upload.wikimedia.org/wikipedia/commons/2/25/Walden_Thoreau.jpg, http://upload.wikimedia.org/wikipedia/commons/b/ba/Henry_David_Thoreau.jpg" +"The Count of Monte Cristo", "Alexandre Dumas", "A man seeks revenge.", "book, classic, France", "http://upload.wikimedia.org/wikipedia/commons/c/c3/Edmond_Dant%C3%A8s.JPG" +"Narrative of the Life of Frederick Douglass", "Frederick Douglass", "A man seeks freedom.", "book, classic, Maryland", "http://upload.wikimedia.org/wikipedia/commons/f/f5/Sketchofdouglass.jpg" diff --git a/csv_files/test_automap_columns_to_elements.csv b/csv_files/test_automap_columns_to_elements.csv new file mode 100644 index 0000000..b71437a --- /dev/null +++ b/csv_files/test_automap_columns_to_elements.csv @@ -0,0 +1,4 @@ +Dublin Core:Title, Dublin Core:Creator, Dublin Core:Description, tags, Item Type Metadata:URL +"Walden", "Henry David Thoreau", "A man seeks simplicity.", "book, classic, New England", "http://upload.wikimedia.org/wikipedia/commons/2/25/Walden_Thoreau.jpg, http://upload.wikimedia.org/wikipedia/commons/b/ba/Henry_David_Thoreau.jpg" +"The Count of Monte Cristo", "Alexandre Dumas", "A man seeks revenge.", "book, classic, France", "http://upload.wikimedia.org/wikipedia/commons/c/c3/Edmond_Dant%C3%A8s.JPG" +"Narrative of the Life of Frederick Douglass", "Frederick Douglass", "A man seeks freedom.", "book, classic, Maryland", "http://upload.wikimedia.org/wikipedia/commons/f/f5/Sketchofdouglass.jpg" diff --git a/forms/Main.php b/forms/Main.php new file mode 100644 index 0000000..c07ee01 --- /dev/null +++ b/forms/Main.php @@ -0,0 +1,453 @@ +_columnDelimiter = CsvImport_RowIterator::getDefaultColumnDelimiter(); + $this->_fileDelimiter = CsvImport_ColumnMap_File::getDefaultFileDelimiter(); + $this->_tagDelimiter = CsvImport_ColumnMap_Tag::getDefaultTagDelimiter(); + $this->_elementDelimiter = CsvImport_ColumnMap_Element::getDefaultElementDelimiter(); + + $this->setAttrib('id', 'csvimport'); + $this->setMethod('post'); + + $this->_addFileElement(); + $values = get_db()->getTable('ItemType')->findPairsForSelectForm(); + $values = array('' => __('Select Item Type')) + $values; + + $this->addElement('checkbox', 'omeka_csv_export', array( + 'label' => __('Use an export from Omeka CSV Report'), + 'description'=> __('Selecting this will override the options below.')) + ); + + $this->addElement('checkbox', 'automap_columns_names_to_elements', array( + 'label' => __('Automap Column Names to Elements'), + 'description'=> __('Automatically maps columns to elements based on their column names. The column name must be in the form:
{ElementSetName}:{ElementName}'), + 'value' => true) + ); + + $this->addElement('select', 'item_type_id', array( + 'label' => __('Select Item Type'), + 'multiOptions' => $values, + )); + $values = get_db()->getTable('Collection')->findPairsForSelectForm(); + $values = array('' => __('Select Collection')) + $values; + + $this->addElement('select', 'collection_id', array( + 'label' => __('Select Collection'), + 'multiOptions' => $values, + )); + $this->addElement('checkbox', 'items_are_public', array( + 'label' => __('Make All Items Public?'), + )); + $this->addElement('checkbox', 'items_are_featured', array( + 'label' => __('Feature All Items?'), + )); + + $this->_addColumnDelimiterElement(); + $this->_addTagDelimiterElement(); + $this->_addFileDelimiterElement(); + $this->_addElementDelimiterElement(); + + $this->applyOmekaStyles(); + $this->setAutoApplyOmekaStyles(false); + + $submit = $this->createElement('submit', + 'submit', + array('label' => __('Next'), + 'class' => 'submit submit-medium')); + + + $submit->setDecorators(array('ViewHelper', + array('HtmlTag', + array('tag' => 'div', + 'class' => 'csvimportnext')))); + + $this->addElement($submit); + } + + /** + * Return the human readable word for a delimiter + * + * @param string $delimiter The delimiter + * @return string The human readable word for the delimiter + */ + protected function _getHumanDelimiterText($delimiter) + { + $delimiterText = $delimiter; + switch ($delimiter) { + case ',': + $delimiterText = __('comma'); + break; + case ';': + $delimiterText = __('semi-colon'); + break; + case '': + $delimiterText = __('empty'); + break; + } + return $delimiterText; + } + + /** + * Add the column delimiter element to the form + */ + protected function _addColumnDelimiterElement() + { + $delimiter = $this->_columnDelimiter; + $humanDelimiterText = $this->_getHumanDelimiterText($delimiter); + $this->addElement('text', 'column_delimiter', array( + 'label' => __('Choose Column Delimiter'), + 'description' => __('A single character that will be used to ' + . 'separate columns in the file (%s by default).' + . ' Note that spaces, tabs, and other whitespace are not accepted.', $humanDelimiterText), + 'value' => $delimiter, + 'required' => true, + 'size' => '1', + 'validators' => array( + array('validator' => 'NotEmpty', + 'breakChainOnFailure' => true, + 'options' => array('messages' => array( + Zend_Validate_NotEmpty::IS_EMPTY => + __('Column delimiter cannot be whitespace and must be one character long.'), + )), + ), + array('validator' => 'StringLength', 'options' => array( + 'min' => 1, + 'max' => 1, + 'messages' => array( + Zend_Validate_StringLength::TOO_SHORT => + __('Column delimiter cannot be whitespace and must be one character long.'), + Zend_Validate_StringLength::TOO_LONG => + __('Column delimiter cannot be whitespace and must be one character long.'), + ), + )), + ), + )); + } + + /** + * Add the file delimiter element to the form + */ + protected function _addFileDelimiterElement() + { + $delimiter = $this->_fileDelimiter; + $humanDelimiterText = $this->_getHumanDelimiterText($delimiter); + $this->addElement('text', 'file_delimiter', array( + 'label' => __('Choose File Delimiter'), + 'description' => __('A single character that will be used to ' + . 'separate file paths or URLs within a cell (%s by default).' + . ' If the delimiter is empty, then the whole text will be used as the file path or URL. Note that spaces, tabs, and other whitespace are not accepted.', $humanDelimiterText), + 'value' => $delimiter, + 'required' => false, + 'size' => '1', + 'validators' => array( + + array('validator' => 'NotEmpty', + 'breakChainOnFailure' => true, + 'options' => array('type' => 'space', 'messages' => array( + Zend_Validate_NotEmpty::IS_EMPTY => + __('File delimiter cannot be whitespace, and must be empty or one character long.'), + )), + ), + + array('validator' => 'StringLength', 'options' => array( + 'min' => 0, + 'max' => 1, + 'messages' => array( + Zend_Validate_StringLength::TOO_SHORT => + __('File delimiter cannot be whitespace, and must be empty or one character long.'), + Zend_Validate_StringLength::TOO_LONG => + __('File delimiter cannot be whitespace, and must be empty or one character long.'), + ), + )), + ), + )); + } + + /** + * Add the tag delimiter element to the form + */ + protected function _addTagDelimiterElement() + { + $delimiter = $this->_tagDelimiter; + $humanDelimiterText = $this->_getHumanDelimiterText($delimiter); + $this->addElement('text', 'tag_delimiter', array( + 'label' => __('Choose Tag Delimiter'), + 'description' => __('A single character that will be used to ' + . 'separate tags within a cell (%s by default).' + . ' Note that spaces, tabs, and other whitespace are not accepted.', $humanDelimiterText), + 'value' => $delimiter, + 'required' => true, + 'size' => '1', + 'validators' => array( + array('validator' => 'NotEmpty', + 'breakChainOnFailure' => true, + 'options' => array('messages' => array( + Zend_Validate_NotEmpty::IS_EMPTY => + __('Tag delimiter cannot be whitespace and must be one character long.'), + )), + ), + array('validator' => 'StringLength', 'options' => array( + 'min' => 1, + 'max' => 1, + 'messages' => array( + Zend_Validate_StringLength::TOO_SHORT => + __('Tag delimiter cannot be whitespace and must be one character long.'), + Zend_Validate_StringLength::TOO_LONG => + __('Tag delimiter cannot be whitespace and must be one character long.'), + ), + )), + ), + )); + } + + /** + * Add the element delimiter element to the form + */ + protected function _addElementDelimiterElement() + { + $delimiter = $this->_elementDelimiter; + $humanDelimiterText = $this->_getHumanDelimiterText($delimiter); + $this->addElement('text', 'element_delimiter', array( + 'label' => __('Choose Element Delimiter'), + 'description' => __('A single character that will be used to ' + . 'separate metadata elements within a cell (%s by default).' + . ' If the delimiter is empty, then the whole text will be used as the element text. Note that spaces, tabs, and other whitespace are not accepted.', $humanDelimiterText), + 'value' => $delimiter, + 'required' => false, + 'size' => '1', + 'validators' => array( + + array('validator' => 'NotEmpty', + 'breakChainOnFailure' => true, + 'options' => array('type' => 'space', 'messages' => array( + Zend_Validate_NotEmpty::IS_EMPTY => + __('Element delimiter cannot be whitespace, and must be empty or one character long.'), + )), + ), + + array('validator' => 'StringLength', 'options' => array( + 'min' => 0, + 'max' => 1, + 'messages' => array( + Zend_Validate_StringLength::TOO_SHORT => + __('Element delimiter cannot be whitespace, and must be empty or one character long.'), + Zend_Validate_StringLength::TOO_LONG => + __('Element delimiter cannot be whitespace, and must be empty or one character long.'), + ), + )), + ), + )); + } + + /** + * Add the file element to the form + */ + protected function _addFileElement() + { + $size = $this->getMaxFileSize(); + $byteSize = clone $this->getMaxFileSize(); + $byteSize->setType(Zend_Measure_Binary::BYTE); + + $fileValidators = array( + new Zend_Validate_File_Size(array( + 'max' => $byteSize->getValue())), + new Zend_Validate_File_Count(1), + ); + if ($this->_requiredExtensions) { + $fileValidators[] = + new Omeka_Validate_File_Extension($this->_requiredExtensions); + } + if ($this->_requiredMimeTypes) { + $fileValidators[] = + new Omeka_Validate_File_MimeType($this->_requiredMimeTypes); + } + // Random filename in the temporary directory. + // Prevents race condition. + $filter = new Zend_Filter_File_Rename($this->_fileDestinationDir + . '/' . md5(mt_rand() + microtime(true))); + $this->addElement('file', 'csv_file', array( + 'label' => __('Upload CSV File'), + 'required' => true, + 'validators' => $fileValidators, + 'destination' => $this->_fileDestinationDir, + 'description' => __("Maximum file size is %s.", $size->toString()) + )); + $this->csv_file->addFilter($filter); + } + + /** + * Validate the form post + */ + public function isValid($post) + { + // Too much POST data, return with an error. + if (empty($post) && (int)$_SERVER['CONTENT_LENGTH'] > 0) { + $maxSize = $this->getMaxFileSize()->toString(); + $this->csv_file->addError( + __('The file you have uploaded exceeds the maximum post size ' + . 'allowed by the server. Please upload a file smaller ' + . 'than %s.', $maxSize)); + return false; + } + + return parent::isValid($post); + } + + /** + * Set the column delimiter for the form. + * + * @param string $delimiter The column delimiter + */ + public function setColumnDelimiter($delimiter) + { + $this->_columnDelimiter = $delimiter; + } + + /** + * Set the file delimiter for the form. + * + * @param string $delimiter The file delimiter + */ + public function setFileDelimiter($delimiter) + { + $this->_fileDelimiter = $delimiter; + } + + /** + * Set the tag delimiter for the form. + * + * @param string $delimiter The tag delimiter + */ + public function setTagDelimiter($delimiter) + { + $this->_tagDelimiter = $delimiter; + } + + /** + * Set the element delimiter for the form. + * + * @param string $delimiter The element delimiter + */ + public function setElementDelimiter($delimiter) + { + $this->_elementDelimiter = $delimiter; + } + + /** + * Set the file destination for the form. + * + * @param string $dest The file destination + */ + public function setFileDestination($dest) + { + $this->_fileDestinationDir = $dest; + } + + /** + * Set the maximum size for an uploaded CSV file. + * + * If this is not set in the plugin configuration, + * defaults to the smaller of 'upload_max_filesize' and 'post_max_size' + * settings in php. + * + * If this is set but it exceeds the aforementioned php setting, the size + * will be reduced to that lower setting. + * + * @param string|null $size The maximum file size + */ + public function setMaxFileSize($size = null) + { + $postMaxSize = $this->_getBinarySize(ini_get('post_max_size')); + $fileMaxSize = $this->_getBinarySize(ini_get('upload_max_filesize')); + + // Start with the max size as the lower of the two php ini settings. + $strictMaxSize = $postMaxSize->compare($fileMaxSize) > 0 + ? $fileMaxSize + : $postMaxSize; + + // If the plugin max file size setting is lower, choose it as the strict max size + $pluginMaxSizeRaw = trim(get_option(CsvImportPlugin::MEMORY_LIMIT_OPTION_NAME)); + if ($pluginMaxSizeRaw != '') { + if ($pluginMaxSize = $this->_getBinarySize($pluginMaxSizeRaw)) { + $strictMaxSize = $strictMaxSize->compare($pluginMaxSize) > 0 + ? $pluginMaxSize + : $strictMaxSize; + } + } + + if ($size === null) { + $maxSize = $this->_maxFileSize; + } else { + $maxSize = $this->_getBinarySize($size); + } + + if ($maxSize === false || + $maxSize === null || + $maxSize->compare($strictMaxSize) > 0) { + $maxSize = $strictMaxSize; + } + + $this->_maxFileSize = $maxSize; + } + + /** + * Return the max file size + * + * @return string The max file size + */ + public function getMaxFileSize() + { + if (!$this->_maxFileSize) { + $this->setMaxFileSize(); + } + return $this->_maxFileSize; + } + + /** + * Return the binary size measure + * + * @return Zend_Measure_Binary The binary size + */ + protected function _getBinarySize($size) + { + if (!preg_match('/(\d+)([KMG]?)/i', $size, $matches)) { + return false; + } + + $sizeType = Zend_Measure_Binary::BYTE; + + $sizeTypes = array( + 'K' => Zend_Measure_Binary::KILOBYTE, + 'M' => Zend_Measure_Binary::MEGABYTE, + 'G' => Zend_Measure_Binary::GIGABYTE, + ); + + if (count($matches) == 3 && array_key_exists($matches[2], $sizeTypes)) { + $sizeType = $sizeTypes[$matches[2]]; + } + + return new Zend_Measure_Binary($matches[1], $sizeType); + } +} diff --git a/forms/Mapping.php b/forms/Mapping.php new file mode 100644 index 0000000..2783b42 --- /dev/null +++ b/forms/Mapping.php @@ -0,0 +1,321 @@ +setAttrib('id', 'csvimport-mapping'); + $this->setMethod('post'); + + $elementsByElementSetName = $this->_getElementPairs($this->_itemTypeId); + $elementsByElementSetName = array('' => 'Select Below') + + $elementsByElementSetName; + foreach ($this->_columnNames as $index => $colName) { + $rowSubForm = new Zend_Form_SubForm(); + $selectElement = $rowSubForm->createElement('select', + 'element', + array( + 'class' => 'map-element', + 'multiOptions' => $elementsByElementSetName, + 'multiple' => false // see ZF-8452 + ) + ); + $selectElement->setIsArray(true); + if ($this->_automapColumnNamesToElements) { + $selectElement->setValue($this->_getElementIdFromColumnName($colName)); + } + + $rowSubForm->addElement($selectElement); + $rowSubForm->addElement('checkbox', 'html'); + $rowSubForm->addElement('checkbox', 'tags'); + $rowSubForm->addElement('checkbox', 'file'); + $this->_setSubFormDecorators($rowSubForm); + $this->addSubForm($rowSubForm, "row$index"); + } + + $this->addElement('submit', 'submit', + array('label' => __('Import CSV File'), + 'class' => 'submit submit-medium')); + } + + protected function _getElementIdFromColumnName($columnName, $columnNameDelimiter=':') + { + $element = $this->_getElementFromColumnName($columnName, $columnNameDelimiter); + if ($element) { + return $element->id; + } else { + return null; + } + } + + /** + * Return the element from the column name + * + * @param string $columnName The name of the column + * @param string $columnNameDelimiter The column name delimiter + * @return Element|null The element from the column name + */ + protected function _getElementFromColumnName($columnName, $columnNameDelimiter=':') + { + $element = null; + // $columnNameParts is an array like array('Element Set Name', 'Element Name') + if (strlen($columnNameDelimiter) > 0) { + if ($columnNameParts = explode($columnNameDelimiter, $columnName)) { + if (count($columnNameParts) == 2) { + $elementSetName = trim($columnNameParts[0]); + $elementName = trim($columnNameParts[1]); + $element = get_db()->getTable('Element') + ->findByElementSetNameAndElementName($elementSetName, $elementName); + } + } + } + return $element; + } + + + /** + * Load the default decorators. + */ + public function loadDefaultDecorators() + { + $this->setDecorators(array( + array('ViewScript', array( + 'viewScript' => 'index/map-columns-form.php', + 'itemTypeId' => $this->_itemTypeId, + 'form' => $this, + 'columnExamples' => $this->_columnExamples, + 'columnNames' => $this->_columnNames, + )), + )); + } + + /** + * Set the column names + * + * @param array $columnNames The array of column names (which are strings) + */ + public function setColumnNames($columnNames) + { + $this->_columnNames = $columnNames; + } + + /** + * Set the column examples + * + * @param array $columnExamples The array of column examples (which are strings) + */ + public function setColumnExamples($columnExamples) + { + $this->_columnExamples = $columnExamples; + } + + /** + * Set the column examples + * + * @param int $itemTypeId The id of the item type + */ + public function setItemTypeId($itemTypeId) + { + $this->_itemTypeId = $itemTypeId; + } + + /** + * Set the element delimiter + * + * @param int $elementDelimiter The element delimiter + */ + public function setElementDelimiter($elementDelimiter) + { + $this->_elementDelimiter = $elementDelimiter; + } + + /** + * Set the file delimiter + * + * @param int $fileDelimiter The file delimiter + */ + public function setFileDelimiter($fileDelimiter) + { + $this->_fileDelimiter = $fileDelimiter; + } + + /** + * Set the tag delimiter + * + * @param int $tagDelimiter The tag delimiter + */ + public function setTagDelimiter($tagDelimiter) + { + $this->_tagDelimiter = $tagDelimiter; + } + + /** + * Set whether or not to automap column names to elements + * + * @param boolean $flag Whether or not to automap column names to elements + */ + public function setAutomapColumnNamesToElements($flag) + { + $this->_automapColumnNamesToElements = (boolean)$flag; + } + + /** + * Returns array of column maps + * + * @return array The array of column maps + */ + public function getColumnMaps() + { + $columnMaps = array(); + foreach ($this->_columnNames as $key => $colName) { + if ($map = $this->_getColumnMap($key, $colName)) { + if (is_array($map)) { + $columnMaps = array_merge($columnMaps, $map); + } else { + $columnMaps[] = $map; + } + } + } + return $columnMaps; + } + + /** + * Returns whether a subform row contains a tag mapping + * + * @param int $index The subform row index + * @return bool Whether the subform row contains a tag mapping + */ + protected function _isTagMapped($index) + { + return $this->getSubForm("row$index")->tags->isChecked(); + } + + /** + * Returns whether a subform row contains a file mapping + * + * @param int $index The subform row index + * @return bool Whether a subform row contains a file mapping + */ + protected function _isFileMapped($index) + { + return $this->getSubForm("row$index")->file->isChecked(); + } + + /** + * Returns the element id mapped to the subform row + * + * @param int $index The subform row index + * @return mixed The element id mapped to the subform row + */ + protected function _getMappedElementId($index) + { + return $this->_getRowValue($index, 'element'); + } + + /** + * Returns a row element value + * + * @param int $index The subform row index + * @param string $elementName The element name in the row + * @return mixed The row element value + */ + protected function _getRowValue($index, $elementName) + { + return $this->getSubForm("row$index")->$elementName->getValue(); + } + + /** + * Adds decorators to a subform. + * + * @param Zend_Form_SubForm $subForm The subform + */ + protected function _setSubFormDecorators($subForm) + { + // Get rid of the fieldset tag that wraps subforms by default. + $subForm->setDecorators(array( + 'FormElements', + )); + + // Each subform is a row in the table. + foreach ($subForm->getElements() as $el) { + $el->setDecorators(array( + array('decorator' => 'ViewHelper'), + array('decorator' => 'HtmlTag', + 'options' => array('tag' => 'td')), + )); + } + } + + /** + * Get the mappings from one column in the CSV file. + * + * Some columns can have multiple mappings; these are represented + * as an array of maps. + * + * @param int $index The subform row index + * @param string $columnName The name of the CSV file column + * @return CsvImport_ColumnMap|array|null A ColumnMap or an array of ColumnMaps + */ + protected function _getColumnMap($index, $columnName) + { + $columnMap = array(); + + if ($this->_isTagMapped($index)) { + $columnMap[] = new CsvImport_ColumnMap_Tag($columnName, $this->_tagDelimiter); + } + + if ($this->_isFileMapped($index)) { + $columnMap[] = new CsvImport_ColumnMap_File($columnName, $this->_fileDelimiter); + } + + $elementIds = $this->_getMappedElementId($index); + $isHtml = $this->_getRowValue($index, 'html'); + foreach($elementIds as $elementId) { + // Make sure to skip empty mappings + if (!$elementId) { + continue; + } + + $elementMap = new CsvImport_ColumnMap_Element($columnName, $this->_elementDelimiter); + $elementMap->setOptions(array('elementId' => $elementId, + 'isHtml' => $isHtml)); + $columnMap[] = $elementMap; + } + + return $columnMap; + } + + /** + * Returns element selection array for an item type or Dublin Core. + * This is used for selecting elements in form dropdowns + * + * @param int|null $itemTypeId The id of the item type. + * If null, then it only includes Dublin Core elements + * @return array + */ + protected function _getElementPairs($itemTypeId=null) + { + $params = $itemTypeId ? array('item_type_id' => $itemTypeId) + : array('exclude_item_type' => true); + return get_db()->getTable('Element')->findPairsForSelectForm($params); + } +} diff --git a/languages/bg_BG.mo b/languages/bg_BG.mo new file mode 100644 index 0000000..0feeb0f Binary files /dev/null and b/languages/bg_BG.mo differ diff --git a/languages/ca_ES.mo b/languages/ca_ES.mo new file mode 100644 index 0000000..b634c81 Binary files /dev/null and b/languages/ca_ES.mo differ diff --git a/languages/cs.mo b/languages/cs.mo new file mode 100644 index 0000000..c401a59 Binary files /dev/null and b/languages/cs.mo differ diff --git a/languages/de_DE.mo b/languages/de_DE.mo new file mode 100644 index 0000000..fad7c8a Binary files /dev/null and b/languages/de_DE.mo differ diff --git a/languages/es.mo b/languages/es.mo new file mode 100644 index 0000000..eeccdb1 Binary files /dev/null and b/languages/es.mo differ diff --git a/languages/et.mo b/languages/et.mo new file mode 100644 index 0000000..e88f4c0 Binary files /dev/null and b/languages/et.mo differ diff --git a/languages/fi_FI.mo b/languages/fi_FI.mo new file mode 100644 index 0000000..463f7f2 Binary files /dev/null and b/languages/fi_FI.mo differ diff --git a/languages/fr.mo b/languages/fr.mo new file mode 100644 index 0000000..e1dbbdd Binary files /dev/null and b/languages/fr.mo differ diff --git a/languages/hr.mo b/languages/hr.mo new file mode 100644 index 0000000..297e19e Binary files /dev/null and b/languages/hr.mo differ diff --git a/languages/it.mo b/languages/it.mo new file mode 100644 index 0000000..e9d0c3e Binary files /dev/null and b/languages/it.mo differ diff --git a/languages/mn.mo b/languages/mn.mo new file mode 100644 index 0000000..d25329b Binary files /dev/null and b/languages/mn.mo differ diff --git a/languages/nl_NL.mo b/languages/nl_NL.mo new file mode 100644 index 0000000..b7b7f5a Binary files /dev/null and b/languages/nl_NL.mo differ diff --git a/languages/pl.mo b/languages/pl.mo new file mode 100644 index 0000000..a1feb2d Binary files /dev/null and b/languages/pl.mo differ diff --git a/languages/pt_BR.mo b/languages/pt_BR.mo new file mode 100644 index 0000000..a381af5 Binary files /dev/null and b/languages/pt_BR.mo differ diff --git a/languages/pt_PT.mo b/languages/pt_PT.mo new file mode 100644 index 0000000..c8e0261 Binary files /dev/null and b/languages/pt_PT.mo differ diff --git a/languages/ro.mo b/languages/ro.mo new file mode 100644 index 0000000..2f8680b Binary files /dev/null and b/languages/ro.mo differ diff --git a/languages/sr_RS.mo b/languages/sr_RS.mo new file mode 100644 index 0000000..8590457 Binary files /dev/null and b/languages/sr_RS.mo differ diff --git a/models/CsvImport/ColumnMap.php b/models/CsvImport/ColumnMap.php new file mode 100644 index 0000000..7833fc0 --- /dev/null +++ b/models/CsvImport/ColumnMap.php @@ -0,0 +1,51 @@ +_columnName = $columnName; + } + + /** + * Returns the type of column map + * + * @return string The type of column map + */ + public function getType() + { + return $this->_type; + } + + /** + * Use the column mapping to convert a CSV row into a value that can be + * parsed by insert_item() or insert_files_for_item(). + * + * @param array $row The row in the CSV file + * @param array $result + * @return array An array value that can be parsed + * by insert_item() or insert_files_for_item() + */ + abstract public function map($row, $result); +} diff --git a/models/CsvImport/ColumnMap/Collection.php b/models/CsvImport/ColumnMap/Collection.php new file mode 100644 index 0000000..b1572d1 --- /dev/null +++ b/models/CsvImport/ColumnMap/Collection.php @@ -0,0 +1,69 @@ +_type = CsvImport_ColumnMap::TYPE_COLLECTION; + } + + /** + * Map a row to an array that can be parsed by + * insert_item() or insert_files_for_item(). + * + * @param array $row The row to map + * @param array $result + * @return array The result + */ + public function map($row, $result) + { + $result = null; + $collectionTitle = $row[$this->_columnName]; + if ($collectionTitle != '') { + $collection = $this->_getCollectionByTitle($collectionTitle); + if ($collection) { + $result = $collection->id; + } + } + return $result; + } + + /** + * Return a collection by its title + * + * @param string $name The collection name + * @return Collection The collection + */ + protected function _getCollectionByTitle($name) + { + $db = get_db(); + + $elementTable = $db->getTable('Element'); + $element = $elementTable->findByElementSetNameAndElementName('Dublin Core', 'Title'); + + $collectionTable = $db->getTable('Collection'); + $select = $collectionTable->getSelect(); + $select->joinInner(array('s' => $db->ElementText), + 's.record_id = collections.id', array()); + $select->where("s.record_type = 'Collection'"); + $select->where("s.element_id = ?", $element->id); + $select->where("s.text = ?", $name); + + $collection = $collectionTable->fetchObject($select); + if (!$collection) { + _log("Collection not found. Collections must be created with identical names prior to import", Zend_Log::NOTICE); + return false; + } + return $collection; + } +} diff --git a/models/CsvImport/ColumnMap/Element.php b/models/CsvImport/ColumnMap/Element.php new file mode 100644 index 0000000..6a2a468 --- /dev/null +++ b/models/CsvImport/ColumnMap/Element.php @@ -0,0 +1,119 @@ +_type = CsvImport_ColumnMap::TYPE_ELEMENT; + if ($elementDelimiter !== null) { + $this->_elementDelimiter = $elementDelimiter; + } else { + $this->_elementDelimiter = self::getDefaultElementDelimiter(); + } + } + + /** + * Map a row to an array that can be parsed by insert_item() or + * insert_files_for_item(). + * + * @param array $row The row to map + * @param array $result + * @return array The result + */ + public function map($row, $result) + { + if ($this->_isHtml) { + $filter = new Omeka_Filter_HtmlPurifier(); + $text = $filter->filter($row[$this->_columnName]); + } else { + $text = $row[$this->_columnName]; + } + if ($this->_elementDelimiter == '') { + $texts = array($text); + } else { + $texts = explode($this->_elementDelimiter, $text); + } + foreach($texts as $text) { + $result[] = array( + 'element_id' => $this->_elementId, + 'html' => $this->_isHtml ? 1 : 0, + 'text' => $text, + ); + } + return $result; + } + + /** + * Sets the mapping options. + * + * @param array $options + */ + public function setOptions($options) + { + $this->_elementId = $options['elementId']; + $this->_isHtml = (boolean)$options['isHtml']; + } + + /** + * Return the element delimiter. + * + * @return string The element delimiter + */ + public function getElementDelimiter() + { + return $this->_elementDelimiter; + } + + /** + * Return the element id. + * + * @return int The element id + */ + public function getElementId() + { + return $this->_elementId; + } + + /** + * Return whether the element texts are HTML. + * + * @return bool Whether the element texts are HTML + */ + public function isHtml() + { + return $this->_isHtml; + } + + /** + * Returns the default element delimiter. + * Uses the default element delimiter specified in the options table if + * available. + * + * @return string The default element delimiter + */ + static public function getDefaultElementDelimiter() + { + if (!($delimiter = get_option(self::ELEMENT_DELIMITER_OPTION_NAME))) { + $delimiter = self::DEFAULT_ELEMENT_DELIMITER; + } + return $delimiter; + } +} \ No newline at end of file diff --git a/models/CsvImport/ColumnMap/ExportedElement.php b/models/CsvImport/ColumnMap/ExportedElement.php new file mode 100644 index 0000000..eec4fe7 --- /dev/null +++ b/models/CsvImport/ColumnMap/ExportedElement.php @@ -0,0 +1,145 @@ +_type = CsvImport_ColumnMap::TYPE_ELEMENT; + $this->_columnNameDelimiter = self::DEFAULT_COLUMN_NAME_DELIMITER; + $this->_elementDelimiter = self::DEFAULT_ELEMENT_DELIMITER; + $this->_isHtml = true; + + $element = $this->_getElementFromColumnName(); + if ($element) { + $this->_elementId = $element->id; + } else { + $this->_elementId = null; + } + } + + /** + * Map a row to an array that can be parsed by + * insert_item() or insert_files_for_item(). + * + * @param array $row The row to map + * @param array $result + * @return array The result + */ + public function map($row, $result) + { + $filter = new Omeka_Filter_HtmlPurifier(); + $text = $filter->filter($row[$this->_columnName]); + if ($this->_elementDelimiter == '') { + $texts = array($text); + } else { + $texts = explode($this->_elementDelimiter, $text); + } + + if ($this->_elementId) { + foreach($texts as $text) { + $result[] = array('element_id' => $this->_elementId, + 'html' => $this->_isHtml ? 1 : 0, + 'text' => $text); + } + } + + return $result; + } + + /** + * Return the element from the column name + * + * @return Element|null The element from the column name + */ + protected function _getElementFromColumnName() + { + $element = null; + // $columnNameParts is an array like array('Element Set Name', 'Element Name') + if (strlen($this->_columnNameDelimiter) > 0) { + if ($columnNameParts = explode($this->_columnNameDelimiter, $this->_columnName)) { + if (count($columnNameParts) == 2) { + $elementSetName = $columnNameParts[0]; + $elementName = $columnNameParts[1]; + $element = get_db()->getTable('Element') + ->findByElementSetNameAndElementName($elementSetName, $elementName); + } + } + } + return $element; + } + + /** + * Sets the mapping options + * + * @param array $options + */ + public function setOptions($options) + { + $this->_columnNameDelimiter = $options['columnNameDelimiter']; + $this->_elementDelimiter = $options['elementDelimiter']; + $this->_elementId = $options['elementId']; + $this->_isHtml = $options['isHtml']; + } + + /** + * Return the element delimiter + * + * @return string The element delimiter + */ + public function getElementDelimiter() + { + return $this->_elementDelimiter; + } + + /** + * Return the column name delimiter + * + * @return string The column name delimiter + */ + public function getColumnNameDelimiter() + { + return $this->_columnNameDelimiter; + } + + /** + * Return the element id + * + * @return int The element id + */ + public function getElementId() + { + return $this->_elementId; + } + + /** + * Return whether the element texts are HTML + * + * @return bool Whether the element texts are HTML + */ + public function isHtml() + { + return $this->_isHtml; + } +} diff --git a/models/CsvImport/ColumnMap/Featured.php b/models/CsvImport/ColumnMap/Featured.php new file mode 100644 index 0000000..acc0cb6 --- /dev/null +++ b/models/CsvImport/ColumnMap/Featured.php @@ -0,0 +1,39 @@ +_type = CsvImport_ColumnMap::TYPE_FEATURED; + } + + /** + * Map a row to whether the row corresponding to an item is featured or not + * + * @param array $row The row to map + * @param array $result + * @return bool Whether the row corresponding to an item is featured or not + */ + public function map($row, $result) + { + $filter = new Omeka_Filter_Boolean; + $flag = strtolower(trim($row[$this->_columnName])); + if ($flag == 'no') { + return 0; + } else if ($flag == 'yes') { + return 1; + } else { + return $filter->filter($flag); + } + } +} \ No newline at end of file diff --git a/models/CsvImport/ColumnMap/File.php b/models/CsvImport/ColumnMap/File.php new file mode 100644 index 0000000..d33823e --- /dev/null +++ b/models/CsvImport/ColumnMap/File.php @@ -0,0 +1,80 @@ +_type = CsvImport_ColumnMap::TYPE_FILE; + if ($fileDelimiter !== null) { + $this->_fileDelimiter = $fileDelimiter; + } else { + $this->_fileDelimiter = self::getDefaultFileDelimiter(); + } + } + + /** + * Map a row to an array that can be parsed by insert_item() or + * insert_files_for_item(). + * + * @param array $row The row to map + * @param array $result + * @return array The result + */ + public function map($row, $result) + { + $urlString = trim($row[$this->_columnName]); + if ($urlString) { + if ($this->_fileDelimiter == '') { + $rawUrls = array($urlString); + } else { + $rawUrls = explode($this->_fileDelimiter, $urlString); + } + $trimmedUrls = array_map('trim', $rawUrls); + $cleanedUrls = array_diff($trimmedUrls, array('')); + $result = array_merge($result, $cleanedUrls); + $result = array_unique($result); + } + return $result; + } + + /** + * Return the file delimiter. + * + * @return string The file delimiter + */ + public function getFileDelimiter() + { + return $this->_fileDelimiter; + } + + /** + * Returns the default file delimiter. + * Uses the default file delimiter specified in the options table if + * available. + * + * @return string The default file delimiter + */ + static public function getDefaultFileDelimiter() + { + if (!($delimiter = get_option(self::FILE_DELIMITER_OPTION_NAME))) { + $delimiter = self::DEFAULT_FILE_DELIMITER; + } + return $delimiter; + } +} diff --git a/models/CsvImport/ColumnMap/ItemType.php b/models/CsvImport/ColumnMap/ItemType.php new file mode 100644 index 0000000..2fca225 --- /dev/null +++ b/models/CsvImport/ColumnMap/ItemType.php @@ -0,0 +1,33 @@ +_type = CsvImport_ColumnMap::TYPE_ITEM_TYPE; + } + + /** + * Map a row to an array that can be parsed by + * insert_item() or insert_files_for_item(). + * + * @param array $row The row to map + * @param array $result + * @return array The result + */ + public function map($row, $result) + { + $result = $row[$this->_columnName]; + return $result; + } +} \ No newline at end of file diff --git a/models/CsvImport/ColumnMap/Public.php b/models/CsvImport/ColumnMap/Public.php new file mode 100644 index 0000000..9440194 --- /dev/null +++ b/models/CsvImport/ColumnMap/Public.php @@ -0,0 +1,39 @@ +_type = CsvImport_ColumnMap::TYPE_PUBLIC; + } + + /** + * Map a row to whether the row corresponding to an item is public or not + * + * @param array $row The row to map + * @param array $result + * @return bool Whether the row corresponding to an item is public or not + */ + public function map($row, $result) + { + $filter = new Omeka_Filter_Boolean; + $flag = strtolower(trim($row[$this->_columnName])); + if ($flag == 'no') { + return 0; + } else if ($flag == 'yes') { + return 1; + } else { + return $filter->filter($flag); + } + } +} \ No newline at end of file diff --git a/models/CsvImport/ColumnMap/Set.php b/models/CsvImport/ColumnMap/Set.php new file mode 100644 index 0000000..42cb92a --- /dev/null +++ b/models/CsvImport/ColumnMap/Set.php @@ -0,0 +1,56 @@ +_maps = $maps; + } + + /** + * Adds a column map to the set + * + * @param CsvImport_ColumnMap $map The column map + */ + public function add(CsvImport_ColumnMap $map) + { + $this->_maps[] = $map; + } + + /** + * Map a row to an associative array of mappings indexed by column mapping type, + * and where each mapping can be parsed by insert_item() or insert_files_for_item(). + * + * @param array $row The row to map + * @return array The associative array of mappings + */ + public function map(array $row) + { + $allResults = array( + CsvImport_ColumnMap::TYPE_FILE => array(), + CsvImport_ColumnMap::TYPE_ELEMENT => array(), + CsvImport_ColumnMap::TYPE_TAG => array(), + CsvImport_ColumnMap::TYPE_COLLECTION => null, + CsvImport_ColumnMap::TYPE_FEATURED => null, + CsvImport_ColumnMap::TYPE_ITEM_TYPE => null, + CsvImport_ColumnMap::TYPE_PUBLIC => null + + ); + foreach ($this->_maps as $map) { + $subset = $allResults[$map->getType()]; + $allResults[$map->getType()] = $map->map($row, $subset); + } + return $allResults; + } +} \ No newline at end of file diff --git a/models/CsvImport/ColumnMap/Tag.php b/models/CsvImport/ColumnMap/Tag.php new file mode 100644 index 0000000..4155713 --- /dev/null +++ b/models/CsvImport/ColumnMap/Tag.php @@ -0,0 +1,75 @@ +_type = CsvImport_ColumnMap::TYPE_TAG; + if ($tagDelimiter !== null) { + $this->_tagDelimiter = $tagDelimiter; + } else { + $this->_tagDelimiter = self::getDefaultTagDelimiter(); + } + } + + /** + * Map a row to an array of tags. + * + * @param array $row The row to map + * @param array $result + * @return array The array of tags + */ + public function map($row, $result) + { + if ($this->_tagDelimiter == '') { + $rawTags = array($row[$this->_columnName]); + } else { + $rawTags = explode($this->_tagDelimiter, $row[$this->_columnName]); + } + $trimmed = array_map('trim', $rawTags); + $cleaned = array_diff($trimmed, array('')); + $tags = array_merge($result, $cleaned); + return $tags; + } + + /** + * Return the tag delimiter. + * + * @return string The tag delimiter + */ + public function getTagDelimiter() + { + return $this->_tagDelimiter; + } + + /** + * Returns the default tag delimiter. + * Uses the default tag delimiter specified in the options table if + * available. + * + * @return string The default tag delimiter + */ + static public function getDefaultTagDelimiter() + { + if (!($delimiter = get_option(self::TAG_DELIMITER_OPTION_NAME))) { + $delimiter = self::DEFAULT_TAG_DELIMITER; + } + return $delimiter; + } +} diff --git a/models/CsvImport/DuplicateColumnException.php b/models/CsvImport/DuplicateColumnException.php new file mode 100644 index 0000000..a521915 --- /dev/null +++ b/models/CsvImport/DuplicateColumnException.php @@ -0,0 +1,10 @@ +_filePath = $filePath; + if ($columnDelimiter) { + $this->_columnDelimiter = $columnDelimiter; + } + } + + /** + * Absolute path to the file. + * + * @return string + */ + public function getFilePath() + { + return $this->_filePath; + } + + /** + * Get an array of headers for the column names + * + * @return array The array of headers for the column names + */ + public function getColumnNames() + { + if (!$this->_columnNames) { + throw new LogicException("CSV file must be validated before " + . "retrieving the list of columns."); + } + return $this->_columnNames; + } + + /** + * Get an array of example data for the columns. + * + * @return array Examples have the same order as the column names. + */ + public function getColumnExamples() + { + if (!$this->_columnExamples) { + throw new LogicException("CSV file must be validated before " + . "retrieving list of column examples."); + } + return $this->_columnExamples; + } + + /** + * Get an iterator for the rows in the CSV file. + * + * @return CsvImport_RowIterator + */ + public function getIterator() + { + if (!$this->_rowIterator) { + $this->_rowIterator = new CsvImport_RowIterator( + $this->getFilePath(), $this->_columnDelimiter); + } + return $this->_rowIterator; + } + + /** + * Parse metadata. Currently retrieves the column names and an "example" + * row, i.e. the first row after the header. + * + * @return boolean + */ + public function parse() + { + if ($this->_columnNames || $this->_columnExamples) { + throw new RuntimeException('Cannot be parsed twice.'); + } + $rowIterator = $this->getIterator(); + try { + $this->_columnNames = $rowIterator->getColumnNames(); + $this->_columnExamples = $rowIterator->current(); + } catch (CsvImport_DuplicateColumnException $e) { + $this->_parseErrors[] = $e->getMessage() + . ' ' . __('Please ensure that all column names are unique.'); + return false; + } catch (CsvImport_MissingColumnException $e) { + $this->_parseErrors[] = $e->getMessage() + . ' ' . __('Please ensure that the CSV file is formatted correctly' + . ' and contains the expected number of columns for each row.'); + return false; + } + return true; + } + + /** + * Get the error string + * + * @return string + */ + public function getErrorString() + { + return join(' ', $this->_parseErrors); + } +} \ No newline at end of file diff --git a/models/CsvImport/Import.php b/models/CsvImport/Import.php new file mode 100644 index 0000000..2b6797f --- /dev/null +++ b/models/CsvImport/Import.php @@ -0,0 +1,774 @@ +_mixins[] = new Mixin_Timestamp($this, 'added', null); + } + + /** + * Sets whether the imported items are public + * + * @param mixed $flag A boolean representation + */ + public function setItemsArePublic($flag) + { + $booleanFilter = new Omeka_Filter_Boolean; + $this->is_public = $booleanFilter->filter($flag); + } + + /** + * Sets whether the imported items are featured + * + * @param mixed $flag A boolean representation + */ + public function setItemsAreFeatured($flag) + { + $booleanFilter = new Omeka_Filter_Boolean; + $this->is_featured = $booleanFilter->filter($flag); + } + + /** + * Sets the collection id of the collection to which the imported items belong + * + * @param int $id The collection id + */ + public function setCollectionId($id) + { + if(!$id) { + $this->collection_id = null; + } else { + $this->collection_id = (int)$id; + } + } + + /** + * Sets the column delimiter in the imported CSV file + * + * @param string The column delimiter of the imported CSV file + */ + public function setColumnDelimiter($delimiter) + { + $this->delimiter = $delimiter; + } + + /** + * Sets the file path of the imported CSV file + * + * @param string The file path of the imported CSV file + */ + public function setFilePath($path) + { + $this->file_path = $path; + } + + /** + * Sets the original filename of the imported CSV file + * + * @param string The original filename of the imported CSV file + */ + public function setOriginalFilename($filename) + { + $this->original_filename = $filename; + } + + /** + * Sets the item type id of the item type of every imported item + * + * @param int $id The item type id + */ + public function setItemTypeId($id) + { + if(!$id) { + $this->item_type_id = null; + } else { + $this->item_type_id = (int)$id; + } + + } + + /** + * Sets the status of the import + * + * @param string The status of the import + */ + public function setStatus($status) + { + $this->status = (string)$status; + } + + /** + * Sets the user id of the owner of the imported items + * + * @param int $id The user id of the owner of the imported items + */ + public function setOwnerId($id) + { + $this->owner_id = (int)$id; + } + + /** + * Sets whether the import is an Omeka export + * + * @param mixed $flag A boolean representation + */ + public function setIsOmekaExport($flag) + { + $this->_isOmekaExport = $flag; + } + + /** + * Sets the column maps for the import + * + * @param CsvImport_ColumnMap_Set|array $maps The set of column maps + * @throws InvalidArgumentException + */ + public function setColumnMaps($maps) + { + if ($maps instanceof CsvImport_ColumnMap_Set) { + $mapSet = $maps; + } else if (is_array($maps)) { + $mapSet = new CsvImport_ColumnMap_Set($maps); + } else { + throw new InvalidArgumentException("Maps must be either an " + . "array or an instance of CsvImport_ColumnMap_Set."); + } + $this->_columnMaps = $mapSet; + } + + /** + * Set the number of items to create before pausing the import. + * + * Used primarily for performance reasons, i.e. long-running imports may + * time out or hog system resources in such a way that prevents other + * imports from running. When used in conjunction with Omeka_Job and + * resume(), this can be used to spawn multiple sequential jobs for a given + * import. + * + * @param int $size + */ + public function setBatchSize($size) + { + $this->_batchSize = (int)$size; + } + + /** + * Executes before the record is deleted. + * @param array $args + */ + protected function beforeSave($args) + { + $this->serialized_column_maps = serialize($this->getColumnMaps()); + } + + /** + * Executes after the record is deleted. + */ + protected function afterDelete() + { + if (file_exists($this->file_path)) { + unlink($this->file_path); + } + } + + /** + * Returns whether there is an error + * + * @return boolean Whether there is an error + */ + public function isError() + { + return $this->isImportError() || + $this->isUndoImportError() || + $this->isOtherError(); + } + + /** + * Returns whether there is an error with the import process + * + * @return boolean Whether there is an error with the import process + */ + public function isImportError() + { + return $this->status == self::IMPORT_ERROR; + } + + /** + * Returns whether there is an error with the undo import process + * + * @return boolean Whether there is an error with the undo import process + */ + public function isUndoImportError() + { + return $this->status == self::UNDO_IMPORT_ERROR; + } + + /** + * Returns whether there is an error that is neither related to an import nor undo import process + * + * @return boolean Whether there is an error that is neither related to an import nor undo import process + */ + public function isOtherError() + { + return $this->status == self::OTHER_ERROR; + } + + /** + * Returns whether the import is stopped + * + * @return boolean Whether the import is stopped + */ + public function isStopped() + { + return $this->status == self::STOPPED; + } + + /** + * Returns whether the import is queued + * + * @return boolean Whether the import is queued + */ + public function isQueued() + { + return $this->status == self::QUEUED; + } + + /** + * Returns whether the undo import is queued + * + * @return boolean Whether the undo import is queued + */ + public function isQueuedUndo() + { + return $this->status == self::QUEUED_UNDO; + } + + /** + * Returns whether the import is completed + * + * @return boolean Whether the import is completed + */ + public function isCompleted() + { + return $this->status == self::COMPLETED; + } + + /** + * Returns whether the import is undone + * + * @return boolean Whether the import is undone + */ + public function isUndone() + { + return $this->status == self::COMPLETED_UNDO; + } + + /** + * Imports the CSV file. This function can only be run once. + * To import the same csv file, you will have to + * create another instance of CsvImport_Import and run start + * Sets import status to self::IN_PROGRESS + * + * @return boolean Whether the import was successful + */ + public function start() + { + $this->status = self::IN_PROGRESS; + $this->save(); + $this->_log("Started import."); + $this->_importLoop($this->file_position); + return !$this->isError(); + } + + /** + * Completes the import. + * Sets import status to self::COMPLETED + * + * @return boolean Whether the import was successfully completed + */ + public function complete() + { + if ($this->isCompleted()) { + $this->_log("Cannot complete an import that is already completed."); + return false; + } + $this->status = self::COMPLETED; + $this->save(); + $this->_log("Completed importing $this->_importedCount items (skipped " + . "$this->skipped_row_count rows)."); + return true; + } + + /** + * Completes the undo import. + * Sets import status to self::COMPLETED_UNDO + * + * @return boolean Whether the undo import was successfully completed + */ + public function completeUndo() + { + if ($this->isUndone()) { + $this->_log("Cannot complete an undo import that is already undone."); + return false; + } + $this->status = self::COMPLETED_UNDO; + $this->save(); + $this->_log("Completed undoing the import."); + return true; + } + + /** + * Resumes the import. + * Sets import status to self::IN_PROGRESS + * + * @return boolean Whether the import was successful after it was resumed + */ + public function resume() + { + if (!$this->isQueued() && !$this->isQueuedUndo()) { + $this->_log("Cannot resume an import or undo import that has not been queued."); + return false; + } + + $undoImport = $this->isQueuedUndo(); + + if ($this->isQueued()) { + $this->status = self::IN_PROGRESS; + $this->save(); + $this->_log("Resumed import."); + $this->_importLoop($this->file_position); + } else { + $this->status = self::IN_PROGRESS_UNDO; + $this->save(); + $this->_log("Resumed undo import."); + $this->_undoImportLoop(); + } + + return !$this->isError(); + } + + /** + * Stops the import or undo import. + * Sets import status to self::STOPPED + * + * @return boolean Whether the import or undo import was stopped due to an error + */ + public function stop() + { + // If the import or undo import loops were prematurely stopped while in progress, + // then there is an error, otherwise there is no error, i.e. the import + // or undo import was completed + if ($this->status != self::IN_PROGRESS and + $this->status != self::IN_PROGRESS_UNDO) { + return false; // no error + } + + // The import or undo import loop was prematurely stopped + $logMsg = "Stopped import or undo import due to error"; + if ($error = error_get_last()) { + $logMsg .= ": " . $error['message']; + } else { + $logMsg .= '.'; + } + $this->status = self::STOPPED; + $this->save(); + $this->_log($logMsg, Zend_Log::ERR); + return true; // stopped with an error + } + + /** + * Queue the import. + * Sets import status to self::QUEUED + * + * @return boolean Whether the import was successfully queued + */ + public function queue() + { + if ($this->isError()) { + $this->_log("Cannot queue an import that has an error."); + return false; + } + + if ($this->isStopped()) { + $this->_log("Cannot queue an import that has been stopped."); + return false; + } + + if ($this->isCompleted()) { + $this->_log("Cannot queue an import that has been completed."); + return false; + } + + if ($this->isUndone()) { + $this->_log("Cannot queue an import that has been undone."); + return false; + } + + $this->status = self::QUEUED; + $this->save(); + $this->_log("Queued import."); + return true; + } + + /** + * Queue the undo import. + * Sets import status to self::QUEUED_UNDO + * + * @return boolean Whether the undo import was successfully queued + */ + public function queueUndo() + { + if ($this->isUndoImportError()) { + $this->_log("Cannot queue an undo import that has an undo import error."); + return false; + } + + if ($this->isOtherError()) { + $this->_log("Cannot queue an undo import that has an error."); + return false; + } + + if ($this->isStopped()) { + $this->_log("Cannot queue an undo import that has been stopped."); + return false; + } + + if ($this->isUndone()) { + $this->_log("Cannot queue an undo import that has been undone."); + return false; + } + + $this->status = self::QUEUED_UNDO; + $this->save(); + $this->_log("Queued undo import."); + return true; + } + + /** + * Undo the import. + * Sets import status to self::IN_PROGRESS_UNDO and then self::COMPLETED_UNDO + * + * @return boolean Whether the import was successfully undone + */ + public function undo() + { + $this->status = self::IN_PROGRESS_UNDO; + $this->save(); + $this->_log("Started undo import."); + $this->_undoImportLoop(); + return !$this->isError(); + } + + /** + * Returns the CsvImport_File object for the import + * + * @return CsvImport_File + */ + public function getCsvFile() + { + if (empty($this->_csvFile)) { + $this->_csvFile = new CsvImport_File($this->file_path, $this->delimiter); + } + return $this->_csvFile; + } + + /** + * Returns the set of column maps for the import + * + * @throws UnexpectedValueException + * @return CsvImport_ColumnMap_Set The set of column maps for the import + */ + public function getColumnMaps() + { + if ($this->_columnMaps === null) { + $columnMaps = unserialize($this->serialized_column_maps); + if (!($columnMaps instanceof CsvImport_ColumnMap_Set)) { + throw new UnexpectedValueException("Column maps must be " + . "an instance of CsvImport_ColumnMap_Set. Instead, the " + . "following was given: " . var_export($columnMaps, true)); + } + $this->_columnMaps = $columnMaps; + } + return $this->_columnMaps; + } + + /** + * Returns the number of items currently imported. If a user undoes an import, + * this number decreases to the number of items left to remove. + * + * @return int The number of items imported minus the number of items undone + */ + public function getImportedItemCount() + { + $iit = $this->getTable('CsvImport_ImportedItem'); + $sql = $iit->getSelectForCount()->where('`import_id` = ?'); + $importedItemCount = $this->getDb()->fetchOne($sql, array($this->id)); + return $importedItemCount; + } + + /** + * Runs the import loop + * + * @param int $startAt A row number in the CSV file. + * @throws Exception + * @return boolean Whether the import loop was successfully run + */ + protected function _importLoop($startAt = null) + { + try { + register_shutdown_function(array($this, 'stop')); + $rows = $this->getCsvFile()->getIterator(); + $rows->rewind(); + if ($startAt) { + $rows->seek($startAt); + } + $rows->skipInvalidRows(true); + $this->_log("Running item import loop. Memory usage: %memory%"); + while ($rows->valid()) { + $row = $rows->current(); + $index = $rows->key(); + $this->skipped_row_count += $rows->getSkippedCount(); + if ($item = $this->_addItemFromRow($row)) { + release_object($item); + } else { + $this->skipped_item_count++; + $this->_log("Skipped item on row #{$index}.", Zend_Log::WARN); + } + $this->file_position = $this->getCsvFile()->getIterator()->tell(); + if ($this->_batchSize && ($index % $this->_batchSize == 0)) { + $this->_log("Completed importing batch of $this->_batchSize " + . "items. Memory usage: %memory%"); + return $this->queue(); + } + $rows->next(); + } + $this->skipped_row_count += $rows->getSkippedCount(); + return $this->complete(); + } catch (Omeka_Job_Worker_InterruptException $e) { + // Interruptions usually indicate that we should resume from + // the last stopping position. + return $this->queue(); + } catch (Exception $e) { + $this->status = self::IMPORT_ERROR; + $this->save(); + $this->_log($e, Zend_Log::ERR); + throw $e; + } + } + + /** + * Runs the undo import loop + * + * @throws Exception + * @return boolean Whether the undo import loop was successfully run + */ + protected function _undoImportLoop() + { + try { + $itemLimitPerQuery = self::UNDO_IMPORT_ITEM_LIMIT_PER_QUERY; + $batchSize = intval($this->_batchSize); + if ($batchSize > 0) { + $itemLimitPerQuery = min($itemLimitPerQuery, $batchSize); + } + register_shutdown_function(array($this, 'stop')); + $db = $this->getDb(); + $searchSql = "SELECT `item_id` FROM $db->CsvImport_ImportedItem" + . " WHERE `import_id` = " . (int)$this->id + . " LIMIT " . $itemLimitPerQuery; + $it = $this->getTable('Item'); + $deletedItemCount = 0; + while ($itemIds = $db->fetchCol($searchSql)) { + $inClause = 'IN (' . join(', ', $itemIds) . ')'; + $items = $it->fetchObjects( + $it->getSelect()->where("`items`.`id` $inClause")); + $deletedItemIds = array(); + foreach ($items as $item) { + $itemId = $item->id; + $item->delete(); + release_object($item); + $deletedItemIds[] = $itemId; + $deletedItemCount++; + if ($batchSize > 0 && $deletedItemCount == $batchSize) { + $inClause = 'IN (' . join(', ', $deletedItemIds) . ')'; + $db->delete($db->CsvImport_ImportedItem, "`item_id` $inClause"); + $this->_log("Completed undoing the import of a batch of $batchSize " + . "items. Memory usage: %memory%"); + return $this->queueUndo(); + } + } + $db->delete($db->CsvImport_ImportedItem, "`item_id` $inClause"); + } + return $this->completeUndo(); + } catch (Omeka_Job_Worker_InterruptException $e) { + if ($db && $deletedItemIds) { + $inClause = 'IN (' . join(', ', $deletedItemIds) . ')'; + $db->delete($db->CsvImport_ImportedItem, "`item_id` $inClause"); + } + return $this->queueUndo(); + } catch (Exception $e) { + $this->status = self::UNDO_IMPORT_ERROR; + $this->save(); + $this->_log($e, Zend_Log::ERR); + throw $e; + } + } + + /** + * Adds a new item based on a row string in the CSV file and returns it. + * + * @param string $row A row string in the CSV file + * @return Item|boolean The inserted item or false if an item could not be added. + */ + protected function _addItemFromRow($row) + { + $result = $this->getColumnMaps()->map($row); + $tags = $result[CsvImport_ColumnMap::TYPE_TAG]; + $itemMetadata = array( + Builder_Item::IS_PUBLIC => $this->is_public, + Builder_Item::IS_FEATURED => $this->is_featured, + Builder_Item::ITEM_TYPE_ID => $this->item_type_id, + Builder_Item::COLLECTION_ID => $this->collection_id, + Builder_Item::TAGS => $tags, + ); + + // If this is coming from CSV Report, bring in the itemmetadata coming from the report + if (!is_null($result[CsvImport_ColumnMap::TYPE_COLLECTION])) { + $itemMetadata[Builder_Item::COLLECTION_ID] = $result[CsvImport_ColumnMap::TYPE_COLLECTION]; + } + if (!is_null($result[CsvImport_ColumnMap::TYPE_PUBLIC])) { + $itemMetadata[Builder_Item::IS_PUBLIC] = $result[CsvImport_ColumnMap::TYPE_PUBLIC]; + } + if (!is_null($result[CsvImport_ColumnMap::TYPE_FEATURED])) { + $itemMetadata[Builder_Item::IS_FEATURED] = $result[CsvImport_ColumnMap::TYPE_FEATURED]; + } + if (!empty($result[CsvImport_ColumnMap::TYPE_ITEM_TYPE])) { + $itemMetadata[Builder_Item::ITEM_TYPE_NAME] = $result[CsvImport_ColumnMap::TYPE_ITEM_TYPE]; + } + + $elementTexts = $result[CsvImport_ColumnMap::TYPE_ELEMENT]; + try { + $item = insert_item($itemMetadata, $elementTexts); + } catch (Omeka_Validator_Exception $e) { + $this->_log($e, Zend_Log::ERR); + return false; + } catch (Omeka_Record_Builder_Exception $e) { + $this->_log($e, Zend_Log::ERR); + return false; + } + + $fileUrls = $result[CsvImport_ColumnMap::TYPE_FILE]; + foreach ($fileUrls as $url) { + try { + $file = insert_files_for_item($item, 'Url', $url, + array('ignore_invalid_files' => false)); + } catch (Omeka_File_Ingest_InvalidException $e) { + $msg = "Invalid file URL '$url': " + . $e->getMessage(); + $this->_log($msg, Zend_Log::ERR); + $item->delete(); + release_object($item); + return false; + } catch (Omeka_File_Ingest_Exception $e) { + $msg = "Could not import file '$url': " + . $e->getMessage(); + $this->_log($msg, Zend_Log::ERR); + $item->delete(); + release_object($item); + return false; + } + release_object($file); + } + + // Makes it easy to unimport the item later. + $this->_recordImportedItemId($item->id); + return $item; + } + + /** + * Records that an item was successfully imported in the database + * + * @param int $itemId The id of the item imported + */ + protected function _recordImportedItemId($itemId) + { + $csvImportedItem = new CsvImport_ImportedItem(); + $csvImportedItem->setArray(array( + 'import_id' => $this->id, + 'item_id' => $itemId) + ); + $csvImportedItem->save(); + $this->_importedCount++; + } + + /** + * Log an import message + * Every message will log the import ID. + * Messages that have %memory% will include memory usage information. + * + * @param string $msg The message to log + * @param int $priority The priority of the message + */ + protected function _log($msg, $priority = Zend_Log::DEBUG) + { + $prefix = "[CsvImport][#{$this->id}]"; + $msg = str_replace('%memory%', memory_get_usage(), $msg); + _log("$prefix $msg", $priority); + } +} diff --git a/models/CsvImport/ImportTask.php b/models/CsvImport/ImportTask.php new file mode 100644 index 0000000..38bfc5a --- /dev/null +++ b/models/CsvImport/ImportTask.php @@ -0,0 +1,103 @@ +_method = self::METHOD_START; + parent::__construct($options); + } + + /** + * Performs the import task + */ + public function perform() + { + if ($this->_memoryLimit) { + ini_set('memory_limit', $this->_memoryLimit); + } + if (!($import = $this->_getImport())) { + return; + } + + $import->setBatchSize($this->_batchSize); + call_user_func(array($import, $this->_method)); + + if ($import->isQueued() || $import->isQueuedUndo()) { + $this->_dispatcher->setQueueName(self::QUEUE_NAME); + $this->_dispatcher->sendLongRunning(__CLASS__, + array( + 'importId' => $import->id, + 'memoryLimit' => $this->_memoryLimit, + 'method' => 'resume', + 'batchSize' => $this->_batchSize, + ) + ); + } + } + + /** + * Set the number of items to create before pausing the import. + * + * @param int $size + */ + public function setBatchSize($size) + { + $this->_batchSize = (int)$size; + } + + /** + * Set the memory limit for the task + * + * @param string $limit + */ + public function setMemoryLimit($limit) + { + $this->_memoryLimit = $limit; + } + + /** + * Set the import id for the task + * + * @param int $id + */ + public function setImportId($id) + { + $this->_importId = (int)$id; + } + + /** + * Set the method name of the import object to be run by the task + * + * @param string $name + */ + public function setMethod($name) + { + $this->_method = $name; + } + + /** + * Returns the import of the import task + * + * @return CsvImport_Import The import of the import task + */ + protected function _getImport() + { + return $this->_db->getTable('CsvImport_Import')->find($this->_importId); + } +} diff --git a/models/CsvImport/ImportedItem.php b/models/CsvImport/ImportedItem.php new file mode 100644 index 0000000..d5233bd --- /dev/null +++ b/models/CsvImport/ImportedItem.php @@ -0,0 +1,34 @@ +item_id; + } + + /** + * Returns the import id for the imported item + * + * @return int The import id. + */ + public function getImportId() + { + return $this->import_id; + } +} diff --git a/models/CsvImport/MissingColumnException.php b/models/CsvImport/MissingColumnException.php new file mode 100644 index 0000000..c0844be --- /dev/null +++ b/models/CsvImport/MissingColumnException.php @@ -0,0 +1,10 @@ +_filePath = $filePath; + if ($columnDelimiter !== null) { + $this->_columnDelimiter = $columnDelimiter; + } else { + $this->_columnDelimiter = self::getDefaultColumnDelimiter(); + } + } + + /** + * Returns the column delimiter. + * + * @return string The column delimiter + */ + public function getColumnDelimiter() + { + return $this->_columnDelimiter; + } + + /** + * Rewind the Iterator to the first element. + * Similar to the reset() function for arrays in PHP. + * + * @throws CsvImport_DuplicateColumnException + */ + public function rewind() + { + if ($this->_handle) { + fclose($this->_handle); + $this->_handle = null; + } + $this->_currentRowNumber = 0; + $this->_valid = true; + // First row should always be the header. + $colRow = $this->_getNextRow(); + $this->_colNames = array_map("trim", array_keys(array_flip($colRow))); + $this->_colCount = count($colRow); + $uniqueColCount = count($this->_colNames); + if ($uniqueColCount != $this->_colCount) { + throw new CsvImport_DuplicateColumnException("Header row " + . "contains $uniqueColCount unique column name(s) for " + . $this->_colCount . " columns."); + } + $this->_moveNext(); + } + + /** + * Return the current element. + * Similar to the current() function for arrays in PHP. + * + * @return mixed current element + */ + public function current() + { + return $this->_currentRow; + } + + /** + * Return the identifying key of the current element. + * Similar to the key() function for arrays in PHP. + * + * @return scalar + */ + public function key() + { + return $this->_currentRowNumber; + } + + /** + * Move forward to next element. + * Similar to the next() function for arrays in PHP. + * + * @throws Exception + */ + public function next() + { + try { + $this->_moveNext(); + } catch (CsvImport_MissingColumnException $e) { + if ($this->_skipInvalidRows) { + $this->_skippedRowCount++; + $this->next(); + } else { + throw $e; + } + } + } + + /** + * Seek to a starting position for the file. + * + * @param int The offset + */ + public function seek($index) + { + if (!$this->_colNames) { + $this->rewind(); + } + $fh = $this->_getFileHandle(); + fseek($fh, $index); + $this->_moveNext(); + } + + /** + * Returns current position of the file pointer. + * + * @return int The current position of the filer pointer + */ + public function tell() + { + return ftell($this->_getFileHandle()); + } + + /** + * Move to the next row in the file. + */ + protected function _moveNext() + { + if ($nextRow = $this->_getNextRow()) { + $this->_currentRow = $this->_formatRow($nextRow); + } else { + $this->_currentRow = array(); + } + + if (!$this->_currentRow) { + fclose($this->_handle); + $this->_valid = false; + $this->_handle = null; + } + } + + /** + * Returns whether the current file position is valid. + * + * @return boolean + */ + public function valid() + { + if (!file_exists($this->_filePath)) { + return false; + } + if (!$this->_getFileHandle()) { + return false; + } + return $this->_valid; + } + + /** + * Returns array of column names. + * + * @return array + */ + public function getColumnNames() + { + if (!$this->_colNames) { + $this->rewind(); + } + return $this->_colNames; + } + + /** + * Returns the number of rows that were skipped since the last time the + * function was called. + * + * Skipped count is reset to 0 after each call to getSkippedCount(). This + * makes it easier to aggregate the number over multiple job runs. + * + * @return int The number of rows skipped since last time function was called + */ + public function getSkippedCount() + { + $skipped = $this->_skippedRowCount; + $this->_skippedRowCount = 0; + return $skipped; + } + + /** + * Sets whether to skip invalid rows. + * + * @param boolean $flag + */ + public function skipInvalidRows($flag) + { + $this->_skipInvalidRows = (boolean)$flag; + } + + /** + * Formats a row. + * + * @throws LogicException + * @throws CsvImport_MissingColumnException + * @return array The formatted row + */ + protected function _formatRow($row) + { + $formattedRow = array(); + if (!isset($this->_colNames)) { + throw new LogicException("Row cannot be formatted until the column " + . "names have been set."); + } + if (count($row) != $this->_colCount) { + $printable = substr(join($this->_columnDelimiter, $row), 0, 30) . '...'; + throw new CsvImport_MissingColumnException("Row beginning with " + . "'$printable' does not have the required {$this->_colCount} " + . "rows."); + } + for ($i = 0; $i < $this->_colCount; $i++) + { + $formattedRow[$this->_colNames[$i]] = $row[$i]; + } + return $formattedRow; + } + + /** + * Returns a file handle for the CSV file. + * + * @return resource The file handle + */ + protected function _getFileHandle() + { + if (!$this->_handle) { + ini_set('auto_detect_line_endings', true); + $this->_handle = fopen($this->_filePath, 'r'); + } + return $this->_handle; + } + + /** + * Returns the next row in the CSV file. + * + * @return array The row + */ + protected function _getNextRow() + { + $currentRow = array(); + $handle = $this->_getFileHandle(); + while (($row = fgetcsv($handle, 0, $this->_columnDelimiter)) !== FALSE) { + $this->_currentRowNumber++; + return $row; + } + } + + /** + * Returns the default column delimiter. + * Uses the default column delimiter specified in the options table if + * available. + * + * @return string The default column delimiter + */ + static public function getDefaultColumnDelimiter() + { + if (!($delimiter = get_option(self::COLUMN_DELIMITER_OPTION_NAME))) { + $delimiter = self::DEFAULT_COLUMN_DELIMITER; + } + return $delimiter; + } +} diff --git a/models/Table/CsvImport/Import.php b/models/Table/CsvImport/Import.php new file mode 100644 index 0000000..407ddf0 --- /dev/null +++ b/models/Table/CsvImport/Import.php @@ -0,0 +1,11 @@ + + 'Import Items', + 'action' => 'index', + 'module' => 'csv-import', + ), + array( + 'label' => 'Status', + 'action' => 'browse', + 'module' => 'csv-import', + ), + ); + echo nav($navArray, 'admin_navigation_settings'); +?> + \ No newline at end of file diff --git a/views/admin/css/csv-import-main.css b/views/admin/css/csv-import-main.css new file mode 100644 index 0000000..50c3824 --- /dev/null +++ b/views/admin/css/csv-import-main.css @@ -0,0 +1,58 @@ +#csvimport { + overflow: visible; +} + +#csv-import-column-mappings-table td { + vertical-align:top; +} + +.csv-import-column-map { + border-top:2px solid #D7D5C4; + padding-top:10px; +} + +#csvimport span.add-element { + background-image: url('../../../../../application/views/scripts/images/silk-icons/add.png'); + background-repeat: no-repeat; + width: 20px; +} + +table#column-mappings div.inputs { + width: auto; +} + + +/* id=section-nav changed to class=section-nav to enable it to work with + * Zend_Navigation. + */ +.horizontal-nav ul.section-nav { + font-size: 1.1em; + margin: 0; + padding: 9px 0 12px; +} + +.horizontal-nav ul.section-nav li { + display: inline; +} + +.horizontal-nav ul.section-nav a { + color: #446677; + padding: 9px 12px 12px; + text-decoration: none; + width: 126px; +} +/* Duplicate of "li.current" definition. Zend_Navigation uses "active" for class + * instead of current. + */ +.horizontal-nav ul.section-nav li.active a { + background: none repeat scroll 0 0 #D7D5C4; + border-color: #D7D5C4; + border-style: solid; + border-width: 0 0 1px; + color: #333333; + font-weight: bold; +} + +div.csvimportnext { + clear:both; +} \ No newline at end of file diff --git a/views/admin/index/browse.php b/views/admin/index/browse.php new file mode 100644 index 0000000..b0698d3 --- /dev/null +++ b/views/admin/index/browse.php @@ -0,0 +1,84 @@ + __('CSV Import'))); +?> + +
+

+ + + + + + + 'th scope="col"', 'list_tag' => '')); + ?> + + + + + + + + + + + getImportedItemCount(); ?> + + + + + + isCompleted() + || $csvImport->isStopped() + || ($csvImport->isImportError() && $importedItemCount > 0)): ?> + url(array('action' => 'undo-import', + 'id' => $csvImport->id), + 'default'); + ?> + + isUndone() || + $csvImport->isUndoImportError() || + $csvImport->isOtherError() || + ($csvImport->isImportError() && $importedItemCount == 0)): ?> + url(array('action' => 'clear-history', + 'id' => $csvImport->id), + 'default'); + ?> + + + + + + + +
added, Zend_Date::DATETIME_SHORT)); ?>original_filename); ?>skipped_item_count); ?>skipped_row_count); ?>status, 'all'))); ?> + + + +
+ +

+ + +
+ + diff --git a/views/admin/index/check-omeka-csv.php b/views/admin/index/check-omeka-csv.php new file mode 100644 index 0000000..852667f --- /dev/null +++ b/views/admin/index/check-omeka-csv.php @@ -0,0 +1,19 @@ + __('CSV Import Errors'))); +?> + +
+

+ +

+

+ +

+

+ +
+ diff --git a/views/admin/index/index.php b/views/admin/index/index.php new file mode 100644 index 0000000..f183fb3 --- /dev/null +++ b/views/admin/index/index.php @@ -0,0 +1,20 @@ + __('CSV Import'))); +?> + +
+ +

+ form; ?> +
+ + diff --git a/views/admin/index/map-columns-form.php b/views/admin/index/map-columns-form.php new file mode 100644 index 0000000..4859e10 --- /dev/null +++ b/views/admin/index/map-columns-form.php @@ -0,0 +1,31 @@ +
+columnNames; + $colExamples = $this->columnExamples; +?> + + + + + + + + + + + + + + + + + + form->getSubForm("row$i"); ?> + + + +
47) { echo '…';} ?>
+
+ form->submit; ?> +
+
diff --git a/views/admin/index/map-columns.php b/views/admin/index/map-columns.php new file mode 100644 index 0000000..b503a38 --- /dev/null +++ b/views/admin/index/map-columns.php @@ -0,0 +1,19 @@ + __('CSV Import'))); +?> + +
+

+ + form; ?> +
+ + diff --git a/views/admin/javascripts/csv-import.js b/views/admin/javascripts/csv-import.js new file mode 100644 index 0000000..7965af1 --- /dev/null +++ b/views/admin/javascripts/csv-import.js @@ -0,0 +1,51 @@ +if (!Omeka) { + var Omeka = {}; +} + +Omeka.CsvImport = {}; + +(function ($) { + /** + * Allow multiple mappings for each field, and add buttons to allow a mapping + * to be removed. + */ + Omeka.CsvImport.enableElementMapping = function () { + $('form#csvimport .map-element').change(function () { + var select = $(this); + var addButton = select.siblings('span.add-element'); + if (!addButton.length) { + var addButton = $(''); + addButton.click(function() { + var copy = select.clone(true); + select.after(copy); + $(this).remove(); + }); + select.after(addButton); + }; + }); + }; + + /** + * Add a confirm step before undoing an import. + */ + Omeka.CsvImport.confirm = function () { + $('.csv-undo-import').click(function () { + return confirm("Undoing an import will delete all of its imported items. Are you sure you want to undo this import?"); + }); + }; + + /** + * Disable most options if Import from Csv Report is checked + */ + Omeka.CsvImport.updateImportOptions = function () { + // we need to test whether the checkbox is checked + // because fields will all be displayed if the form fails validation + var fields = $('div.field').has('#automap_columns_names_to_elements, #item_type_id, #collection_id, #items_are_public, #items_are_featured, #column_delimiter, #element_delimiter, #tag_delimiter, #file_delimiter'); + if ($('#omeka_csv_export').is(':checked')) { + fields.slideUp(); + } else { + fields.slideDown(); + } + }; + +})(jQuery);