diff --git a/ktapi/KTAPIDocument.inc.php b/ktapi/KTAPIDocument.inc.php index 72bcd70..8f19aca 100644 --- a/ktapi/KTAPIDocument.inc.php +++ b/ktapi/KTAPIDocument.inc.php @@ -1071,7 +1071,7 @@ class KTAPI_Document extends KTAPI_FolderItem $packed = $this->get_packed_metadata($metadata); DBUtil::startTransaction(); - $result = KTDocumentUtil::saveMetadata($this->document, $packed); + $result = KTDocumentUtil::saveMetadata($this->document, $packed, array('novalidate'=>true)); if (is_null($result)) { diff --git a/ktapi/KTAPIFolder.inc.php b/ktapi/KTAPIFolder.inc.php index 287a7a7..5e530b0 100644 --- a/ktapi/KTAPIFolder.inc.php +++ b/ktapi/KTAPIFolder.inc.php @@ -610,7 +610,7 @@ class KTAPI_Folder extends KTAPI_FolderItem $options = array( - //'contents' => new KTFSFileLike($tempfilename), + 'contents' => new KTFSFileLike($tempfilename), 'temp_file' => $tempfilename, 'novalidate' => true, 'documenttype' => DocumentType::get($documenttypeid), @@ -622,7 +622,7 @@ class KTAPI_Folder extends KTAPI_FolderItem DBUtil::startTransaction(); $document =& KTDocumentUtil::add($this->folder, $filename, $user, $options); - if (!is_a($document,'Document')) + if (PEAR::isError($document)) { DBUtil::rollback(); return new PEAR_Error(KTAPI_ERROR_INTERNAL_ERROR . ' : ' . $document->getMessage()); diff --git a/ktwebservice/webservice.php b/ktwebservice/webservice.php index ea07b48..c9d5cf4 100644 --- a/ktwebservice/webservice.php +++ b/ktwebservice/webservice.php @@ -1978,6 +1978,7 @@ class KTWebService $status_code = $update_result->value['status_code']; if ($status_code != 0) { + $this->delete_document($session_id, $document_id, 'Rollback because metadata could not be added'); return $update_result; } @@ -2017,6 +2018,7 @@ class KTWebService $status_code = $update_result->value['status_code']; if ($status_code != 0) { + $this->delete_document($session_id, $document_id, 'Rollback because metadata could not be added'); return $update_result; } diff --git a/lib/mime.inc.php b/lib/mime.inc.php index 693d85e..dad1b44 100644 --- a/lib/mime.inc.php +++ b/lib/mime.inc.php @@ -207,6 +207,44 @@ class KTMime { function stripAllButExtension($sFileName) { return strtolower(substr($sFileName, strrpos($sFileName, ".")+1, strlen($sFileName) - strrpos($sFileName, "."))); } + + /** + * getAllMimeTypesInformation is a staic function used to get a fuller set of + * information on the mime types held in the database. + * + */ + function getAllMimeTypesInformation() + { + $sTable = KTUtil::getTableName('mimetypes'); + $aQuery = array('SELECT MT.id, MT.filetypes, MT.mimetypes, MT.icon_path, MT.friendly_name, ME.name as extractor FROM ' + . $sTable .' MT LEFT JOIN mime_extractors ME ON(MT.extractor_id = ME.id) ORDER BY MT.filetypes', array()); + $res = DBUtil::getResultArray($aQuery); + return $res; + } + + /** + * get all information on all the extractors in the database + * + */ + function getMimeExtractorInformation() + { + $aQuery = array('SELECT id, name, active FROM mime_extractors ORDER BY name', array()); + $res = DBUtil::getResultArray($aQuery); + return $res; + } + + /** + *give the mimetype name and get the friendly names and the extensions + * + */ + function getFriendlyNameAndExtension($sMimeType) + { + $sTable = KTUtil::getTableName('mimetypes'); + $sQuery = "SELECT friendly_name, filetypes FROM " . $sTable . " WHERE mimetypes = ?"; + $aQuery = array($sQuery, array($sMimeType)); + $res = DBUtil::getResultArray($aQuery); + return $res; + } } $_KT_icon_path_cache = array(); diff --git a/plugins/ktcore/KTColumns.inc.php b/plugins/ktcore/KTColumns.inc.php index 02fa4d3..5ce1725 100644 --- a/plugins/ktcore/KTColumns.inc.php +++ b/plugins/ktcore/KTColumns.inc.php @@ -5,32 +5,32 @@ * KnowledgeTree Open Source Edition * Document Management Made Simple * Copyright (C) 2004 - 2008 The Jam Warehouse Software (Pty) Limited - * + * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License version 3 as published by the * Free Software Foundation. - * + * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. - * + * * You should have received a copy of the GNU General Public License * along with this program. If not, see . - * + * * You can contact The Jam Warehouse Software (Pty) Limited, Unit 1, Tramber Place, * Blake Street, Observatory, 7925 South Africa. or email info@knowledgetree.com. - * + * * The interactive user interfaces in modified source and object code versions * of this program must display Appropriate Legal Notices, as required under * Section 5 of the GNU General Public License version 3. - * + * * In accordance with Section 7(b) of the GNU General Public License version 3, * these Appropriate Legal Notices must retain the display of the "Powered by - * KnowledgeTree" logo and retain the original copyright notice. If the display of the + * KnowledgeTree" logo and retain the original copyright notice. If the display of the * logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices - * must display the words "Powered by KnowledgeTree" and retain the original - * copyright notice. + * must display the words "Powered by KnowledgeTree" and retain the original + * copyright notice. * Contributor( s): ______________________________________ */ @@ -418,6 +418,36 @@ class AdvancedWorkflowColumn extends AdvancedColumn { } } +class CheckedOutByColumn extends AdvancedColumn { + var $namespace = 'ktcore.columns.checkedout_by'; + var $sortable = false; + + function CheckedOutByColumn() { + $this->label = _kt('Checked Out By'); + $this->sortable = false; + } + + function renderData($aDataRow) { + // only show this for documents. + if ($aDataRow['type'] === 'folder') { + return ' '; + } + + // Check if document is checked out + $bIsCheckedOut = $aDataRow['document']->getIsCheckedOut(); + + if($bIsCheckedOut){ + // Get the user id + $iUserId = $aDataRow['document']->getCheckedOutUserID(); + $oUser = User::get($iUserId); + $sUser = $oUser->getName(); + + return ''.htmlentities($sUser, ENT_NOQUOTES, 'UTF-8').''; + } + return '—'; + } +} + class AdvancedDownloadColumn extends AdvancedColumn { var $namespace = 'ktcore.columns.download'; diff --git a/plugins/ktcore/KTCorePlugin.php b/plugins/ktcore/KTCorePlugin.php index 5e59514..e80bcf4 100644 --- a/plugins/ktcore/KTCorePlugin.php +++ b/plugins/ktcore/KTCorePlugin.php @@ -5,32 +5,32 @@ * KnowledgeTree Open Source Edition * Document Management Made Simple * Copyright (C) 2004 - 2008 The Jam Warehouse Software (Pty) Limited - * + * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License version 3 as published by the * Free Software Foundation. - * + * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. - * + * * You should have received a copy of the GNU General Public License * along with this program. If not, see . - * + * * You can contact The Jam Warehouse Software (Pty) Limited, Unit 1, Tramber Place, * Blake Street, Observatory, 7925 South Africa. or email info@knowledgetree.com. - * + * * The interactive user interfaces in modified source and object code versions * of this program must display Appropriate Legal Notices, as required under * Section 5 of the GNU General Public License version 3. - * + * * In accordance with Section 7(b) of the GNU General Public License version 3, * these Appropriate Legal Notices must retain the display of the "Powered by - * KnowledgeTree" logo and retain the original copyright notice. If the display of the + * KnowledgeTree" logo and retain the original copyright notice. If the display of the * logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices - * must display the words "Powered by KnowledgeTree" and retain the original - * copyright notice. + * must display the words "Powered by KnowledgeTree" and retain the original + * copyright notice. * Contributor( s): ______________________________________ * */ @@ -110,7 +110,7 @@ class KTCorePlugin extends KTPlugin { $this->registerDashlet('IndexingStatusDashlet', 'ktcore.dashlet.indexing_status', KT_DIR . '/plugins/search2/IndexingStatusDashlet.php'); $this->registerDashlet('LuceneStatisticsDashlet', 'ktcore.dashlet.indexing_statss', KT_DIR . '/plugins/search2/LuceneStatisticsDashlet.php'); $this->registerDashlet('schedulerDashlet', 'ktcore.schedulerdashlet.plugin', 'scheduler/schedulerDashlet.php'); - + $this->registerAdminPage('scheduler', 'manageSchedulerDispatcher', 'misc', _kt('Manage Task Scheduler'), _kt('Manage the task scheduler'), 'scheduler/taskScheduler.php'); $this->registerAdminPage('authentication', 'KTAuthenticationAdminPage', 'principals', _kt('Authentication'), sprintf(_kt('By default, %s controls its own users and groups and stores all information about them inside the database. In many situations, an organisation will already have a list of users and groups, and needs to use that existing information to allow access to the DMS. These Authentication Sources allow the system administrator to specify additional sources of authentication data.'), APP_NAME), 'authentication/authenticationadminpage.inc.php'); @@ -134,6 +134,7 @@ class KTCorePlugin extends KTPlugin { $this->registerColumn(_kt('Selection'), 'ktcore.columns.selection', 'AdvancedSelectionColumn', 'KTColumns.inc.php'); $this->registerColumn(_kt('Single Selection'), 'ktcore.columns.singleselection', 'AdvancedSingleSelectionColumn', 'KTColumns.inc.php'); $this->registerColumn(_kt('Workflow State'), 'ktcore.columns.workflow_state', 'AdvancedWorkflowColumn', 'KTColumns.inc.php'); + $this->registerColumn(_kt('Checked Out By'), 'ktcore.columns.checkedout_by', 'CheckedOutByColumn', 'KTColumns.inc.php'); $this->registerColumn(_kt('Creation Date'), 'ktcore.columns.creationdate', 'CreationDateColumn', 'KTColumns.inc.php'); $this->registerColumn(_kt('Modification Date'), 'ktcore.columns.modificationdate', 'ModificationDateColumn', 'KTColumns.inc.php'); $this->registerColumn(_kt('Creator'), 'ktcore.columns.creator', 'CreatorColumn', 'KTColumns.inc.php'); @@ -158,7 +159,7 @@ class KTCorePlugin extends KTPlugin { $this->registerTrigger('edit', 'postValidate', 'SavedSearchSubscriptionTrigger', 'ktcore.search2.savedsearch.subscription.edit', KT_DIR . '/plugins/search2/Search2Triggers.php'); $this->registerTrigger('add', 'postValidate', 'SavedSearchSubscriptionTrigger', 'ktcore.search2.savedsearch.subscription.add', KT_DIR . '/plugins/search2/Search2Triggers.php'); $this->registerTrigger('discussion', 'postValidate', 'SavedSearchSubscriptionTrigger', 'ktcore.search2.savedsearch.subscription.discussion', KT_DIR . '/plugins/search2/Search2Triggers.php'); - + //Tag Cloud Triggers $this->registerTrigger('add', 'postValidate', 'KTAddDocumentTrigger', 'ktcore.triggers.tagcloud.add', KT_DIR.'/plugins/tagcloud/TagCloudTriggers.php'); $this->registerTrigger('edit', 'postValidate', 'KTEditDocumentTrigger', 'ktcore.triggers.tagcloud.edit', KT_DIR.'/plugins/tagcloud/TagCloudTriggers.php'); @@ -182,7 +183,7 @@ class KTCorePlugin extends KTPlugin { $this->registerWidget('KTCoreFolderCollectionWidget', 'ktcore.widgets.foldercollection', 'KTWidgets.php'); $this->registerWidget('KTCoreConditionalSelectionWidget', 'ktcore.widgets.conditionalselection', 'KTWidgets.php'); - + $this->registerPage('collection', 'KTCoreCollectionPage', 'KTWidgets.php'); $this->registerPage('notifications', 'KTNotificationOverflowPage', 'KTMiscPages.php'); @@ -232,6 +233,8 @@ class KTCorePlugin extends KTPlugin { _kt('Manage checked-out, archived and deleted documents.')); $this->registerAdminCategory('documents', _kt('Document Metadata and Workflow Configuration'), _kt('Configure the document metadata: Document Types, Document Fieldsets, Link Types and Workflows.')); + $this->registerAdminCategory('search', _kt('Search and Indexing'), + _kt('Search and Indexing Settings')); $this->registerAdminCategory('misc', _kt('Miscellaneous'), _kt('Various settings which do not fit into the other categories, including managing help and saved searches.')); @@ -289,7 +292,26 @@ class KTCorePlugin extends KTPlugin { _kt('Restore or Expunge Deleted Documents'), _kt('Restore previously deleted documents, or permanently expunge them.'), 'admin/deletedDocuments.php', null); + //Search and Indexing + $this->registerAdminPage('managemimetypes', 'ManageMimeTypesDispatcher', 'search', + _kt('Mime Types'), _kt('This report lists all mime types and extensions that can be identified by KnowledgeTree.'), + '../search2/reporting/ManageMimeTypes.php', null); + + $this->registerAdminPage('extractorinfo', 'ExtractorInfoDispatcher', 'search', + _kt('Extractor Information'), _kt('This report lists the text extractors and their supported mime types.'), + '../search2/reporting/ExtractorInfo.php', null); + + $this->registerAdminPage('indexerrors', 'IndexErrorsDispatcher', 'search', + _kt('Document Indexing Diagnostics'), _kt('This report will help to diagnose problems with document indexing.'), + '../search2/reporting/IndexErrors.php', null); + + $this->registerAdminPage('pendingdocuments', 'PendingDocumentsDispatcher', 'search', + _kt('Pending Documents Indexing Queue'), _kt('This report lists documents that are waiting to be indexed.'), + '../search2/reporting/PendingDocuments.php', null); + $this->registerAdminPage('reschedulealldocuments', 'RescheduleDocumentsDispatcher', 'search', + _kt('Reschedule all documents'), _kt('This function allows you to re-index your entire repository.'), + '../search2/reporting/RescheduleDocuments.php', null); // misc $this->registerAdminPage('helpmanagement', 'ManageHelpDispatcher', 'misc', diff --git a/plugins/search2/reporting/ExtractorInfo.php b/plugins/search2/reporting/ExtractorInfo.php new file mode 100644 index 0000000..0e6fc1d --- /dev/null +++ b/plugins/search2/reporting/ExtractorInfo.php @@ -0,0 +1,129 @@ +. + * + * You can contact The Jam Warehouse Software (Pty) Limited, Unit 1, Tramber Place, + * Blake Street, Observatory, 7925 South Africa. or email info@knowledgetree.com. + * + * The interactive user interfaces in modified source and object code versions + * of this program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU General Public License version 3. + * + * In accordance with Section 7(b) of the GNU General Public License version 3, + * these Appropriate Legal Notices must retain the display of the "Powered by + * KnowledgeTree" logo and retain the original copyright notice. If the display of the + * logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices + * must display the words "Powered by KnowledgeTree" and retain the original + * copyright notice. + * Contributor( s): ______________________________________ + * + */ + +require_once(KT_LIB_DIR . '/dispatcher.inc.php'); +require_once(KT_LIB_DIR . '/templating/templating.inc.php'); +require_once(KT_LIB_DIR . '/mime.inc.php'); + +class ExtractorInfoDispatcher extends KTAdminDispatcher { + + + function check() { + $this->aBreadcrumbs[] = array( + 'url' => $_SERVER['PHP_SELF'], + 'name' => _kt('Extractor Information'), + ); + return parent::check(); + } + + function do_main() { + + //registerTypes registers the mime types and populates the needed tables. + $indexer = Indexer::get(); + $indexer->registerTypes(); + + $oTemplating =& KTTemplating::getSingleton(); + $oTemplating->addLocation('Extractor Information', '/plugins/search2/reporting/templates'); + + $oTemplate =& $oTemplating->loadTemplate('extractorinfo'); + + $aExtractorInfo = KTMime::getMimeExtractorInformation(); + + if(empty($aExtractorInfo)) + { + $oTemplate->setData(array( + 'context' => $this, + 'extractor_info' => $aExtractorInfo + )); + + return $oTemplate; + } + + foreach($aExtractorInfo as $key=>$info) + { + $extractorClass = $info['name']; + $extractor = $indexer->getExtractor($extractorClass); + $info['mimeTypes'] = array(); + $aMimeTypes = $this->getSupportedMimeTypesDB($extractorClass);//$extractor->getSupportedMimeTypes(); + + + foreach($aMimeTypes as $mimeType) + { + $sMimeInfo = KTMime::getFriendlyNameAndExtension($mimeType); + + $info['mimeTypes'][$mimeType] = array('description'=>$sMimeInfo[0]['friendly_name'], 'extensions'=>array($sMimeInfo[0]['filetypes'])); + + $extensions = array(); + foreach($sMimeInfo as $item) + { + $extensions[] = $item['filetypes']; + } + $info['mimeTypes'][$mimeType]['extensions'] = implode(', ', $extensions); + } + + $aExtractorInfo[$key] = $info; + } + + $oTemplate->setData(array( + 'context' => $this, + 'extractor_info' => $aExtractorInfo + )); + return $oTemplate; + } + + function getSupportedMimeTypesDB($sExtractorName) + { + $sQuery = "SELECT MT.mimetypes FROM mime_extractors as ME LEFT JOIN mime_types as MT ON " . + "(ME.id = MT.extractor_id) WHERE ME.name = ?"; + $aQuery = array($sQuery, array($sExtractorName)); + $aTempRes = DBUtil::getResultArray($aQuery); + $aRes = array(); + for($i = 0; $i < count($aTempRes); $i++ ) + { + if(!in_array($aTempRes[$i]['mimetypes'], $aRes)) + { + $aRes[] = $aTempRes[$i]['mimetypes']; + + } + } + return $aRes; + } + +} + + +?> diff --git a/plugins/search2/reporting/IndexErrors.php b/plugins/search2/reporting/IndexErrors.php new file mode 100644 index 0000000..0f25265 --- /dev/null +++ b/plugins/search2/reporting/IndexErrors.php @@ -0,0 +1,98 @@ +. + * + * You can contact The Jam Warehouse Software (Pty) Limited, Unit 1, Tramber Place, + * Blake Street, Observatory, 7925 South Africa. or email info@knowledgetree.com. + * + * The interactive user interfaces in modified source and object code versions + * of this program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU General Public License version 3. + * + * In accordance with Section 7(b) of the GNU General Public License version 3, + * these Appropriate Legal Notices must retain the display of the "Powered by + * KnowledgeTree" logo and retain the original copyright notice. If the display of the + * logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices + * must display the words "Powered by KnowledgeTree" and retain the original + * copyright notice. + * Contributor( s): ______________________________________ + * + */ + +require_once(KT_LIB_DIR . '/dispatcher.inc.php'); +require_once(KT_LIB_DIR . '/templating/templating.inc.php'); +require_once(KT_LIB_DIR . '/mime.inc.php'); + +class IndexErrorsDispatcher extends KTAdminDispatcher { + + function check() { + $this->aBreadcrumbs[] = array( + 'url' => $_SERVER['PHP_SELF'], + 'name' => _kt('Document Indexing Diagnostics'), + ); + return parent::check(); + } + + function do_main() { + + //registerTypes registers the mime types and populates the needed tables. + $indexer = Indexer::get(); + $indexer->registerTypes(); + + if($_REQUEST['rescheduleValue'] == 'reschedule') + { + + foreach(KTUtil::arrayGet($_REQUEST, 'index_error', array()) as $sDocId => $v) + { + Indexer::reindexDocument($sDocId); + + } + + } + else if($_REQUEST['rescheduleValue'] == 'rescheduleall') + { + $aIndexerValues = Indexer::getIndexingQueue(); + foreach ($aIndexerValues as $sDocValues) + { + Indexer::reindexDocument($sDocValues['document_id']); + } + + } + require_once(KT_LIB_DIR . "/templating/templating.inc.php"); + $oTemplating =& KTTemplating::getSingleton(); + $oTemplating->addLocation('Index Errors', '/plugins/search2/reporting/templates'); + + $oTemplate =& $oTemplating->loadTemplate('indexerrors'); + + $aIndexerValues = Indexer::getIndexingQueue(); + + $oTemplate->setData(array( + 'context' => $this, + 'index_errors' => $aIndexerValues + + )); + return $oTemplate; + } + + + +} + + +?> diff --git a/plugins/search2/reporting/ManageMimeTypes.php b/plugins/search2/reporting/ManageMimeTypes.php new file mode 100644 index 0000000..cea5ff4 --- /dev/null +++ b/plugins/search2/reporting/ManageMimeTypes.php @@ -0,0 +1,98 @@ +. + * + * You can contact The Jam Warehouse Software (Pty) Limited, Unit 1, Tramber Place, + * Blake Street, Observatory, 7925 South Africa. or email info@knowledgetree.com. + * + * The interactive user interfaces in modified source and object code versions + * of this program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU General Public License version 3. + * + * In accordance with Section 7(b) of the GNU General Public License version 3, + * these Appropriate Legal Notices must retain the display of the "Powered by + * KnowledgeTree" logo and retain the original copyright notice. If the display of the + * logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices + * must display the words "Powered by KnowledgeTree" and retain the original + * copyright notice. + * Contributor( s): ______________________________________ + * + */ +require_once(KT_LIB_DIR . '/dispatcher.inc.php'); +require_once(KT_LIB_DIR . '/templating/templating.inc.php'); +require_once(KT_LIB_DIR . '/mime.inc.php'); + +class ManageMimeTypesDispatcher extends KTAdminDispatcher { + + function check() { + $this->aBreadcrumbs[] = array( + 'url' => $_SERVER['PHP_SELF'], + 'name' => _kt('Manage Mime Types'), + ); + return parent::check(); + } + + function do_main() { + + //registerTypes registers the mime types and populates the needed tables. + $indexer = Indexer::get(); + $indexer->registerTypes(); + + $oTemplating =& KTTemplating::getSingleton(); + $oTemplating->addLocation('Manage Mime Type Plugin', '/plugins/search2/reporting/templates'); + + $oTemplate =& $oTemplating->loadTemplate('managemimetypes'); + + $aMimeTypes = KTMime::getAllMimeTypesInformation(); + + $indexer = Indexer::get(); + + $numExtensions = 0; + $numIndexedExtensions = 0; + + foreach($aMimeTypes as $key=>$mimeType) + { + $extractorClass = $mimeType['extractor']; + $numExtensions++; + if (empty($extractorClass)) + { + continue; + } + $extractor = $indexer->getExtractor($extractorClass); + $aMimeTypes[$key]['extractor'] = $extractor->getDisplayName(); + $numIndexedExtensions++; + } + + $indexedPercentage = 0; + if ($numExtensions > 0) + { + $indexedPercentage = number_format(($numIndexedExtensions * 100)/$numExtensions,2,'.',','); + } + + $oTemplate->setData(array( + 'context' => $this, + 'mime_types' => $aMimeTypes, + 'numExtensions'=>$numExtensions, + 'numIndexedExtensions'=>$numIndexedExtensions, + 'indexedPercentage'=>$indexedPercentage + + )); + return $oTemplate; + } +} \ No newline at end of file diff --git a/plugins/search2/reporting/PendingDocuments.php b/plugins/search2/reporting/PendingDocuments.php new file mode 100644 index 0000000..d13cbe7 --- /dev/null +++ b/plugins/search2/reporting/PendingDocuments.php @@ -0,0 +1,74 @@ +. + * + * You can contact The Jam Warehouse Software (Pty) Limited, Unit 1, Tramber Place, + * Blake Street, Observatory, 7925 South Africa. or email info@knowledgetree.com. + * + * The interactive user interfaces in modified source and object code versions + * of this program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU General Public License version 3. + * + * In accordance with Section 7(b) of the GNU General Public License version 3, + * these Appropriate Legal Notices must retain the display of the "Powered by + * KnowledgeTree" logo and retain the original copyright notice. If the display of the + * logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices + * must display the words "Powered by KnowledgeTree" and retain the original + * copyright notice. + * Contributor( s): ______________________________________ + * + */ + +require_once(KT_LIB_DIR . '/dispatcher.inc.php'); +require_once(KT_LIB_DIR . '/templating/templating.inc.php'); + +class PendingDocumentsDispatcher extends KTAdminDispatcher +{ + function check() { + $this->aBreadcrumbs[] = array( + 'url' => $_SERVER['PHP_SELF'], + 'name' => _kt('Pending Documents Indexing Queue'), + ); + return parent::check(); + } + + function do_main() { + + //registerTypes registers the mime types and populates the needed tables. + $indexer = Indexer::get(); + $indexer->registerTypes(); + + $aPendingDocs = Indexer::getPendingIndexingQueue(); + + $oTemplating =& KTTemplating::getSingleton(); + $oTemplating->addLocation('Pending Documents', '/plugins/search2/reporting/templates'); + $oTemplate =& $oTemplating->loadTemplate('pendingdocuments'); + + $oTemplate->setData(array( + 'context' => $this, + 'pending_docs' => $aPendingDocs + + )); + return $oTemplate; + } + +} + + +?> diff --git a/plugins/search2/reporting/RescheduleDocuments.php b/plugins/search2/reporting/RescheduleDocuments.php new file mode 100644 index 0000000..f9eb65f --- /dev/null +++ b/plugins/search2/reporting/RescheduleDocuments.php @@ -0,0 +1,82 @@ +. + * + * You can contact The Jam Warehouse Software (Pty) Limited, Unit 1, Tramber Place, + * Blake Street, Observatory, 7925 South Africa. or email info@knowledgetree.com. + * + * The interactive user interfaces in modified source and object code versions + * of this program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU General Public License version 3. + * + * In accordance with Section 7(b) of the GNU General Public License version 3, + * these Appropriate Legal Notices must retain the display of the "Powered by + * KnowledgeTree" logo and retain the original copyright notice. If the display of the + * logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices + * must display the words "Powered by KnowledgeTree" and retain the original + * copyright notice. + * Contributor( s): ______________________________________ + * + */ + +require_once(KT_LIB_DIR . '/dispatcher.inc.php'); +require_once(KT_LIB_DIR . '/templating/templating.inc.php'); + +class RescheduleDocumentsDispatcher extends KTAdminDispatcher +{ + function check() { + $this->aBreadcrumbs[] = array( + 'url' => $_SERVER['PHP_SELF'], + 'name' => _kt('Reschedule all documents'), + ); + return parent::check(); + } + + function do_main() { + + //registerTypes registers the mime types and populates the needed tables. + $indexer = Indexer::get(); + $indexer->registerTypes(); + + $oTemplating =& KTTemplating::getSingleton(); + $oTemplating->addLocation('Reschedule Documents', '/plugins/search2/reporting/templates'); + + + $oTemplate =& $oTemplating->loadTemplate('rescheduledocuments'); + + if ($_REQUEST['rescheduleValue'] == 'reschedule') + { + Indexer::indexAll(); + $oTemplate->setData(array( + 'context' => $this, + 'rescheduleDone' => true + )); + return $oTemplate; + } + + $oTemplate->setData(array( + 'context' => $this, + 'rescheduleDone' => false + + )); + return $oTemplate; + } +} + +?> diff --git a/plugins/search2/reporting/templates/extractorinfo.smarty b/plugins/search2/reporting/templates/extractorinfo.smarty new file mode 100644 index 0000000..220f6fe --- /dev/null +++ b/plugins/search2/reporting/templates/extractorinfo.smarty @@ -0,0 +1,39 @@ +

{i18n}Extractor Information{/i18n}

+

{i18n}This report lists the text extractors and their supported mime types.{/i18n}

+ +{if $extractor_info} + +{foreach key=key from=$extractor_info item=extractor} +
+

{$extractor.name}{if $extractor.active == 1} ({i18n}Active{/i18n}){else} ({i18n}Inactive{/i18n}){/if}

+ + + + + + + + + + + + + +{foreach from=$extractor.mimeTypes key=key item=mimeType} + + + + + +{/foreach} + + + +
{i18n}Description{/i18n}{i18n}Mime Types{/i18n}{i18n}Extensions{/i18n}
{$mimeType.description}{$key}{$extractor.mimeTypes.$key.extensions}
+ +{/foreach} + +{else} +
+

{i18n}There are no extractors registered.{/i18n}

+{/if} diff --git a/plugins/search2/reporting/templates/indexerrors.smarty b/plugins/search2/reporting/templates/indexerrors.smarty new file mode 100644 index 0000000..b120ea5 --- /dev/null +++ b/plugins/search2/reporting/templates/indexerrors.smarty @@ -0,0 +1,65 @@ +{literal} + +{/literal} + +

{i18n}Document Indexing Diagnostics{/i18n}

+

{i18n}This report will help to diagnose problems with document indexing.{/i18n}

+ +
+{if $index_errors} + +
+ + + + + + + + + + + + + + + + +{foreach key=key from=$index_errors item=indexError} + + + + + + + + + + + + + + + + +{/foreach} + + + +
{i18n}Document ID{/i18n}{i18n}Filename{/i18n}{i18n}Extension{/i18n}{i18n}Mime Type{/i18n}{i18n}Extractor{/i18n}{i18n}Index Date{/i18n}
{$indexError.document_id}{$indexError.filename|truncate:40:'...'}{$indexError.filetypes}{$indexError.mimetypes}{if $pendingDocs.extractor}{$indexError.extractor}{else}

{i18n}n/a{/i18n}

{/if}		{$indexError.indexdate}
 
{$indexError.status_msg}
+ +
+ + +{else} +

{i18n}There are no indexing issues.{/i18n}

+{/if} +
\ No newline at end of file diff --git a/plugins/search2/reporting/templates/managemimetypes.smarty b/plugins/search2/reporting/templates/managemimetypes.smarty new file mode 100644 index 0000000..862c72f --- /dev/null +++ b/plugins/search2/reporting/templates/managemimetypes.smarty @@ -0,0 +1,48 @@ +

{i18n}Manage Mime Types{/i18n}

+

{i18n}This report lists all mime types and extensions that can be identified by KnowledgeTree.{/i18n}

+ + +{if $mime_types} + + + + + + + + + + + + + + +{foreach from=$mime_types item=mimetype} + + + + + + + +{/foreach} + + +
Icon Extension Mime Type Description Extractor
{$mimetype.filetypes}{$mimetype.mimetypes}{if $mimetype.friendly_name} +{$mimetype.friendly_name} +{else} +

no description

+{/if}		{if $mimetype.extractor} +{$mimetype.extractor} +{else} +

n/a

+{/if}
+ + +
Number of Extensions{$numExtensions} +
Number of indexed extensions{$numIndexedExtensions} ( {$indexedPercentage}% ) +
+ + +{/if} + diff --git a/plugins/search2/reporting/templates/pendingdocuments.smarty b/plugins/search2/reporting/templates/pendingdocuments.smarty new file mode 100644 index 0000000..6aa2b03 --- /dev/null +++ b/plugins/search2/reporting/templates/pendingdocuments.smarty @@ -0,0 +1,44 @@ +

{i18n}Pending Documents Indexing Queue{/i18n}

+{i18n}This report lists documents that are waiting to be indexed.{/i18n} +

+{i18n}If a document is not associated with an extractor, no content will be added to the index. These documents can be identified in the list by the extractor column reflecting n/a.{/i18n} +

+ +{if empty($pending_docs)} + +{i18n}There are no documents in the indexing queue.{/i18n} + +{else} + + + + + + + + + + + + + + + + + +{foreach key=key from=$pending_docs item=pendingDocs} + + + + + + + + +{/foreach} + + + +
{i18n}Document ID{/i18n}{i18n}Filename{/i18n}{i18n}Extension{/i18n}{i18n}Mime Type{/i18n}{i18n}Extractor{/i18n}{i18n}Index Date{/i18n}
{$pendingDocs.document_id}{$pendingDocs.filename|truncate:40:'...'}{$pendingDocs.filetypes}{$pendingDocs.mimetypes}{if $pendingDocs.extractor}{$pendingDocs.extractor}{else}

{i18n}n/a{/i18n}

{/if}		{$pendingDocs.indexdate}
+ +{/if} \ No newline at end of file diff --git a/plugins/search2/reporting/templates/rescheduledocuments.smarty b/plugins/search2/reporting/templates/rescheduledocuments.smarty new file mode 100644 index 0000000..9038956 --- /dev/null +++ b/plugins/search2/reporting/templates/rescheduledocuments.smarty @@ -0,0 +1,15 @@ +

{i18n}Reschedule All Documents{/i18n}

+

{i18n}This function allows you to re-index your entire repository.{/i18n}

+

{i18n}Please note that rescheduling all documents may take a long time, depending on the size if the repository.{/i18n} + +

+
+{if $rescheduleDone == true} +

Documents Rescheduled

+

You can view the schedule queue here

+
+{/if} + + + +
\ No newline at end of file diff --git a/search2/indexing/indexerCore.inc.php b/search2/indexing/indexerCore.inc.php index 50af536..f458902 100644 --- a/search2/indexing/indexerCore.inc.php +++ b/search2/indexing/indexerCore.inc.php @@ -36,6 +36,7 @@ * */ +define('SEARCH2_INDEXER_DIR',realpath(dirname(__FILE__)) . '/'); require_once('indexing/extractorCore.inc.php'); require_once(KT_DIR . '/plugins/ktcore/scheduler/schedulerUtil.php'); @@ -492,6 +493,19 @@ abstract class Indexer $default->log->debug("index: Queuing indexing of $document_id"); } + public static function reindexQueue() + { + $sql = "UPDATE index_files SET processdate = null"; + DBUtil::runQuery($sql); + } + + public static function reindexDocument($documentId) + { + $sql = "UPDATE index_files SET processdate=null, status_msg=null WHERE document_id=$documentId"; + DBUtil::runQuery($sql); + } + + public static function indexAll() { @@ -741,6 +755,107 @@ abstract class Indexer KTUtil::setSystemSetting('mimeTypesRegistered', true); } + private function updatePendingDocumentStatus($documentId, $message, $level) + { + $this->indexingHistory .= "\n" . $level . ': ' . $message; + $message = sanitizeForSQL($this->indexingHistory); + $sql = "UPDATE index_files SET status_msg='$message' WHERE document_id=$documentId"; + DBUtil::runQuery($sql); + } + + /** + * + * @param int $documentId + * @param string $message + * @param string $level This may be info, error, debug + */ + private function logPendingDocumentInfoStatus($documentId, $message, $level) + { + $this->updatePendingDocumentStatus($documentId, $message, $level); + global $default; + + switch ($level) + { + case 'debug': + if ($this->debug) + { + $default->log->debug($message); + } + break; + default: + $default->log->$level($message); + } + } + + + + public function getExtractor($extractorClass) + { + $includeFile = SEARCH2_INDEXER_DIR . 'extractors/' . $extractorClass . '.inc.php'; + if (!file_exists($includeFile)) + { + throw new Exception("Extractor file does not exist: $includeFile"); + } + + require_once($includeFile); + + if (!class_exists($extractorClass)) + { + throw new Exception("Extractor '$classname' not defined in file: $includeFile"); + } + + $extractor = new $extractorClass(); + + if (!($extractor instanceof DocumentExtractor)) + { + throw new Exception("Class $classname was expected to be of type DocumentExtractor"); + } + + return $extractor; + } + + public static function getIndexingQueue($problemItemsOnly=true) + { + + if ($problemItemsOnly) + { + $sql = "SELECT + iff.document_id, iff.indexdate, mt.filetypes, mt.mimetypes, me.name as extractor, iff.what, iff.status_msg, dcv.filename + FROM + index_files iff + INNER JOIN documents d ON iff.document_id=d.id + INNER JOIN document_metadata_version dmv ON d.metadata_version_id=dmv.id + INNER JOIN document_content_version dcv ON dmv.content_version_id=dcv.id + INNER JOIN mime_types mt ON dcv.mime_id=mt.id + LEFT JOIN mime_extractors me ON mt.extractor_id=me.id + WHERE + (iff.status_msg IS NOT NULL) AND dmv.status_id=1 + ORDER BY indexdate "; + } + else + { + $sql = "SELECT + iff.document_id, iff.indexdate, mt.filetypes, mt.mimetypes, me.name as extractor, iff.what, iff.status_msg, dcv.filename + FROM + index_files iff + INNER JOIN documents d ON iff.document_id=d.id + INNER JOIN document_metadata_version dmv ON d.metadata_version_id=dmv.id + INNER JOIN document_content_version dcv ON dmv.content_version_id=dcv.id + INNER JOIN mime_types mt ON dcv.mime_id=mt.id + LEFT JOIN mime_extractors me ON mt.extractor_id=me.id + WHERE + (iff.status_msg IS NULL or iff.status_msg = '') AND dmv.status_id=1 + ORDER BY indexdate "; + } + $aResult = DBUtil::getResultArray($sql); + + return $aResult; + } + + public static function getPendingIndexingQueue() + { + return Indexer::getIndexingQueue(false); + } /** * The main function that may be called repeatedly to index documents. @@ -841,11 +956,10 @@ abstract class Indexer $extractorClass=$docinfo['extractor']; $indexDocument = in_array($docinfo['what'], array('A','C')); $indexDiscussion = in_array($docinfo['what'], array('A','D')); + $this->indexingHistory = ''; + + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Indexing docid: %d extension: '%s' mimetype: '%s' extractor: '%s'"), $docId, $extension,$mimeType,$extractorClass), 'debug'); - if ($this->debug) - { - if ($this->debug) $default->log->debug(sprintf(_kt("Indexing docid: %d extension: '%s' mimetype: '%s' extractor: '%s'"), $docId, $extension,$mimeType,$extractorClass)); - } if (empty($extractorClass)) { @@ -855,13 +969,13 @@ abstract class Indexer if (!$this->isExtractorEnabled($extractorClass)) { - $default->log->info(sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass)); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info'); continue; } if ($this->debug) { - $default->log->info(sprintf(_kt("Processing docid: %d.\n"),$docId)); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Processing docid: %d.\n"),$docId), 'info'); } $removeFromQueue = true; @@ -873,26 +987,12 @@ abstract class Indexer } else { - require_once('extractors/' . $extractorClass . '.inc.php'); - - if (!class_exists($extractorClass)) - { - $default->log->error(sprintf(_kt("indexDocuments: extractor '%s' does not exist."),$extractorClass)); - continue; - } - - $extractor = $extractorCache[$extractorClass] = new $extractorClass(); - } - - if (is_null($extractor)) - { - $default->log->error(sprintf(_kt("indexDocuments: extractor '%s' not resolved - it is null."),$extractorClass)); - continue; + $extractor = $extractorCache[$extractorClass] = $this->getExtractor($extractorClass); } if (!($extractor instanceof DocumentExtractor)) { - $default->log->error(sprintf(_kt("indexDocuments: extractor '%s' is not a document extractor class."),$extractorClass)); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("indexDocuments: extractor '%s' is not a document extractor class."),$extractorClass), 'error'); continue; } @@ -914,7 +1014,7 @@ abstract class Indexer $result = @copy($sourceFile, $intermediate); if ($result === false) { - $default->log->error(sprintf(_kt("Could not create intermediate file from document %d"),$docId)); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Could not create intermediate file from document %d"),$docId), 'error'); // problem. lets try again later. probably permission related. log the issue. continue; } @@ -931,7 +1031,7 @@ abstract class Indexer $extractor->setIndexingStatus(null); $extractor->setExtractionStatus(null); - if ($this->debug) $default->log->debug(sprintf(_kt("Extra Info docid: %d Source File: '%s' Target File: '%s'"),$docId,$sourceFile,$targetFile)); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Extra Info docid: %d Source File: '%s' Target File: '%s'"),$docId,$sourceFile,$targetFile), 'debug'); $this->executeHook($extractor, 'pre_extract'); $this->executeHook($extractor, 'pre_extract', $mimeType); @@ -952,7 +1052,8 @@ abstract class Indexer if (!$indexStatus) { - $default->log->error(sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId)); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId), 'error'); + } $extractor->setIndexingStatus($indexStatus); @@ -961,7 +1062,7 @@ abstract class Indexer { if (!$this->filterText($targetFile)) { - $default->log->error(sprintf(_kt("Problem filtering document %d"),$docId)); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"),$docId), 'error'); } else { @@ -969,7 +1070,8 @@ abstract class Indexer if (!$indexStatus) { - $default->log->error(sprintf(_kt("Problem indexing document %d - indexDocument"),$docId)); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocument"),$docId), 'error'); + $this->logPendingDocumentInfoStatus($docId, '' . $extractor->output . '', 'error'); } $extractor->setIndexingStatus($indexStatus); @@ -982,7 +1084,7 @@ abstract class Indexer else { $extractor->setExtractionStatus(false); - $default->log->error(sprintf(_kt("Could not extract contents from document %d"),$docId)); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Could not extract contents from document %d"),$docId), 'error'); } $this->executeHook($extractor, 'post_extract', $mimeType); diff --git a/sql/mysql/install/structure.sql b/sql/mysql/install/structure.sql index 273e14b..9b2f8fc 100644 --- a/sql/mysql/install/structure.sql +++ b/sql/mysql/install/structure.sql @@ -892,6 +892,7 @@ CREATE TABLE `index_files` ( `indexdate` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, `processdate` datetime default NULL, `what` char(1) default NULL, + `status_msg` mediumtext NULL, PRIMARY KEY (`document_id`), KEY `user_id` (`user_id`), CONSTRAINT `index_files_ibfk_1` FOREIGN KEY (`document_id`) REFERENCES `documents` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, diff --git a/sql/mysql/upgrade/3.5.2/index_file_status_message.sql b/sql/mysql/upgrade/3.5.2/index_file_status_message.sql new file mode 100644 index 0000000..ff436a2 --- /dev/null +++ b/sql/mysql/upgrade/3.5.2/index_file_status_message.sql @@ -0,0 +1 @@ +alter table index_files add status_msg mediumtext; \ No newline at end of file