From 5ed30c96b97d89a1f8a9a81b53d188d2b8fadce1 Mon Sep 17 00:00:00 2001 From: kevin_fourie Date: Fri, 22 Feb 2008 12:22:14 +0000 Subject: [PATCH] Merged in from DEV trunk... --- bin/luceneserver/ktlucene.jar | Bin 12377 -> 0 bytes lib/foldermanagement/compressionArchiveUtil.inc.php | 59 ++++++++++++++++++++++++++++++++++++----------------------- lib/plugins/pluginutil.inc.php | 3 ++- lib/util/ktutil.inc | 2 +- rss.php | 14 ++++++++++---- search2/indexing/extractorCore.inc.php | 9 +++++---- search2/indexing/extractors/OOTextExtractor.inc.php | 11 ++++++----- search2/indexing/indexerCore.inc.php | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++----------------- search2/search/fields/DiscussionTextField.inc.php | 2 ++ 9 files changed, 112 insertions(+), 55 deletions(-) diff --git a/bin/luceneserver/ktlucene.jar b/bin/luceneserver/ktlucene.jar index 96eed59..d3ed923 100644 Binary files a/bin/luceneserver/ktlucene.jar and b/bin/luceneserver/ktlucene.jar differ diff --git a/lib/foldermanagement/compressionArchiveUtil.inc.php b/lib/foldermanagement/compressionArchiveUtil.inc.php index 394429b..9e35401 100644 --- a/lib/foldermanagement/compressionArchiveUtil.inc.php +++ b/lib/foldermanagement/compressionArchiveUtil.inc.php @@ -43,6 +43,8 @@ class ZipFolder { var $sTmpPath = ''; var $sZipFileName = ''; var $sZipFile = ''; + var $sPattern = ''; + var $sFolderPattern = ''; var $aPaths = array(); var $aReplaceKeys = array(); var $aReplaceValues = array(); @@ -55,6 +57,8 @@ class ZipFolder { $this->oKTConfig =& KTConfig::getSingleton(); $this->oStorage =& KTStorageManagerUtil::getSingleton(); + $this->sOutputEncoding = $this->oKTConfig->get('export/encoding', 'UTF-8'); + $sBasedir = $this->oKTConfig->get("urls/tmpDirectory"); $sTmpPath = tempnam($sBasedir, 'kt_compress_zip'); @@ -65,6 +69,9 @@ class ZipFolder { $this->sZipFileName = $sZipFileName; $this->aPaths = array(); + $this->sPattern = "[\*|\%|\\\|\/|\<|\>|\+|\:|\?|\||\'|\"]"; + $this->sFolderPattern = "[\*|\%|\<|\>|\+|\:|\?|\||\'|\"]"; + $aReplace = array( "[" => "[[]", " " => "[ ]", @@ -76,7 +83,7 @@ class ZipFolder { $this->aReplaceValues = array_values($aReplace); } - /** + /** * Return the full path * * @param mixed $oFolderOrDocument May be a Folder or Document @@ -109,28 +116,30 @@ class ZipFolder { } $sDocPath = $this->getFullFolderPath($oFolder); + $sDocPath = preg_replace($this->sFolderPattern, '-', $sDocPath); + $sDocPath = $this->_convertEncoding($sDocPath, true); + + $sDocName = $oDocument->getFileName(); + $sDocName = preg_replace($this->sPattern, '-', $sDocName); + $sDocName = $this->_convertEncoding($sDocName, true); - $sParentFolder = str_replace('<', '', str_replace('', '', sprintf('%s/%s', $this->sTmpPath, $sDocPath)))); + $sParentFolder = $this->sTmpPath.'/'.$sDocPath; $newDir = $this->sTmpPath; - $sFullPath = str_replace('<', '', str_replace('', '', $this->_convertEncoding($sDocPath, true)))); - foreach (split('/', $sFullPath) as $dirPart) { + + $aFullPath = split('/', $sDocPath); + foreach ($aFullPath as $dirPart) { $newDir = sprintf("%s/%s", $newDir, $dirPart); if (!file_exists($newDir)) { mkdir($newDir, 0700); } } - $sOrigFile = str_replace('<', '', str_replace('', '', $this->oStorage->temporaryFile($oDocument)))); - $sFilename = sprintf("%s/%s", $sParentFolder, str_replace('<', '', str_replace('', '', $sDocName)))); - $sFilename = $this->_convertEncoding($sFilename, true); + $sOrigFile = $this->oStorage->temporaryFile($oDocument); + $sFilename = $sParentFolder.'/'.$sDocName; copy($sOrigFile, $sFilename); - $sPath = str_replace('<', '', str_replace('', '', sprintf("%s/%s", $sDocPath, $sDocName)))); - $sPath = str_replace($this->aReplaceKeys, $this->aReplaceValues, $sPath); - $sPath = $this->_convertEncoding($sPath, true); - - $this->aPaths[] = $sPath; + $this->aPaths[] = $sDocPath.'/'.$sDocName; return true; } @@ -139,21 +148,20 @@ class ZipFolder { */ function addFolderToZip($oFolder) { $sFolderPath = $this->getFullFolderPath($oFolder) .'/'; - $sParentFolder = str_replace('<', '', str_replace('', '', sprintf('%s/%s', $this->sTmpPath, $sFolderPath)))); + $sFolderPath = preg_replace($this->sFolderPattern, '-', $sFolderPath); + $sFolderPath = $this->_convertEncoding($sFolderPath, true); + $newDir = $this->sTmpPath; - $sFullPath = str_replace('<', '', str_replace('', '', $this->_convertEncoding($sFolderPath, true)))); - foreach (split('/', $sFullPath) as $dirPart) { + + $aFullPath = split('/', $sFolderPath); + foreach ($aFullPath as $dirPart) { $newDir = sprintf("%s/%s", $newDir, $dirPart); if (!file_exists($newDir)) { mkdir($newDir, 0700); } } - $sPath = str_replace('<', '', str_replace('', '', sprintf("%s", $sFolderPath)))); - $sPath = str_replace($this->aReplaceKeys, $this->aReplaceValues, $sPath); - $sPath = $this->_convertEncoding($sPath, true); - - $this->aPaths[] = $sPath; + $this->aPaths[] = $sFolderPath; return true; } @@ -163,10 +171,14 @@ class ZipFolder { function createZipFile($bEchoStatus = FALSE) { if(empty($this->aPaths)){ return PEAR::raiseError(_kt("No folders or documents found to compress")); - //$this->addErrorMessage(_kt("No folders or documents found to compress")); - //return false; } + // Set environment language to output character encoding + $loc = $this->sOutputEncoding; + putenv("LANG=$loc"); + putenv("LANGUAGE=$loc"); + $loc = setlocale(LC_ALL, $loc); + $sManifest = sprintf("%s/%s", $this->sTmpPath, "MANIFEST"); file_put_contents($sManifest, join("\n", $this->aPaths)); $sZipFile = sprintf("%s/%s.zip", $this->sTmpPath, $this->sZipFileName); @@ -218,6 +230,7 @@ class ZipFolder { if(!(isset($exportCode) && !empty($exportCode))) { $exportCode = KTUtil::arrayGet($_SESSION['zipcompression'], 'exportcode'); } + $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode); if(!empty($aData)){ @@ -232,7 +245,7 @@ class ZipFolder { return PEAR::raiseError(_kt('The ZIP file can only be downloaded once - if you cancel the download, you will need to reload the page.')); } - header("Content-Type: application/zip"); + header("Content-Type: application/zip; charset=utf-8"); header("Content-Length: ". filesize($sZipFile)); header("Content-Disposition: attachment; filename=\"" . $this->sZipFileName . ".zip" . "\""); header("Expires: Mon, 26 Jul 1997 05:00:00 GMT"); diff --git a/lib/plugins/pluginutil.inc.php b/lib/plugins/pluginutil.inc.php index ed70a97..a052ca1 100644 --- a/lib/plugins/pluginutil.inc.php +++ b/lib/plugins/pluginutil.inc.php @@ -412,7 +412,8 @@ class KTPluginUtil { } } - foreach (KTPluginEntity::getList() as $oPluginEntity) { + $aPluginList = KTPluginEntity::getList(); + foreach ($aPluginList as $oPluginEntity) { $sPath = $oPluginEntity->getPath(); if (!KTUtil::isAbsolutePath($sPath)) { $sPath = sprintf("%s/%s", KT_DIR, $sPath); diff --git a/lib/util/ktutil.inc b/lib/util/ktutil.inc index 1596777..b265414 100644 --- a/lib/util/ktutil.inc +++ b/lib/util/ktutil.inc @@ -786,7 +786,7 @@ class KTUtil { $sPath = str_replace('\\', '/', $sPath); $sReal = str_replace('\\', '/', realpath($sPath)); - return ($sReal == $sPath); + return (strtolower($sReal) == strtolower($sPath)); if (substr($sPath, 0, 1) == '/') { return true; diff --git a/rss.php b/rss.php index a28f131..b8bb12a 100644 --- a/rss.php +++ b/rss.php @@ -74,10 +74,16 @@ if (!validateUser($_SERVER['PHP_AUTH_USER'], $_SERVER['PHP_AUTH_PW'])) { $user = DBAuthenticator::getUser($_SERVER['PHP_AUTH_USER'], array('id'=>'id',)); $id = $user[$_SERVER['PHP_AUTH_USER']]['id']; - header('Content-Type: application/rss+xml; charset=utf-8;'); - header('Content-Disposition: inline; filename="rss.xml"'); - header('Expires: Mon, 26 Jul 1997 05:00:00 GMT'); - header('Last-Modified: ' . gmdate('D, d M Y H:i:s') . ' GMT'); + if(OS_WINDOWS){ + $sReferrer = $_SERVER['HTTP_USER_AGENT']; + // Check if this is IE 6 + if(strstr($sReferrer, 'MSIE 6.0')){ + header('Content-Type: application/rss+xml; charset=utf-8;'); + header('Content-Disposition: inline; filename="rss.xml"'); + header('Expires: Mon, 26 Jul 1997 05:00:00 GMT'); + header('Last-Modified: ' . gmdate('D, d M Y H:i:s') . ' GMT'); + } + } if(KTUtil::arrayGet($_REQUEST, 'docId')){ // if a docId parameter is passed // get document id from http request object diff --git a/search2/indexing/extractorCore.inc.php b/search2/indexing/extractorCore.inc.php index 85a7825..36ec73c 100644 --- a/search2/indexing/extractorCore.inc.php +++ b/search2/indexing/extractorCore.inc.php @@ -370,7 +370,9 @@ abstract class ExternalDocumentExtractor extends DocumentExtractor $script_name = $script_prefix . '.bat'; $script = "rem This is an auto generated file. \n"; - $script .= $cmd . ' 2>>"' . $script_out . '" >>"' . $script_out . "\"\r\n"; + $script .= $cmd . ' 2>"' . $script_out . "\"\r\n"; + $script .= "set er=%ERRORLEVEL%\r\n"; + $script .= "exit /B %er%\r\n"; } else { @@ -392,9 +394,8 @@ abstract class ExternalDocumentExtractor extends DocumentExtractor // execute the script file if (OS_WINDOWS) { - $script_name = "\"$script_name\""; - $WshShell = new COM("WScript.Shell"); - $res = $WshShell->Run($script_name, 0, true); + $res = KTUtil::pexec("\"$script_name\""); + $res = $res['ret']; } else { diff --git a/search2/indexing/extractors/OOTextExtractor.inc.php b/search2/indexing/extractors/OOTextExtractor.inc.php index 71506d3..c26b7b4 100644 --- a/search2/indexing/extractors/OOTextExtractor.inc.php +++ b/search2/indexing/extractors/OOTextExtractor.inc.php @@ -88,16 +88,17 @@ class OOTextExtractor extends ExternalDocumentExtractor protected function getCommandLine() { - //$sourcefile = escapeshellcmd($this->sourcefile); - $sourcefile = ($this->sourcefile); + $sourcefile = $this->sourcefile; + unlink($this->targetfile); $this->targetfile .= '.' . $this->targetExtension; - //$targetfile = escapeshellcmd($this->targetfile); - $targetfile = ($this->targetfile); + $targetfile = $this->targetfile; - $escape = OS_WINDOWS?'"':'\''; + $escape = '"'; $cmdline = "{$escape}{$this->python}{$escape} {$escape}{$this->documentConverter}{$escape} {$escape}{$sourcefile}{$escape} {$escape}{$targetfile}{$escape} {$this->ooHost} {$this->ooPort}"; + $cmdline = str_replace('\\','/',$cmdline); + return $cmdline; } diff --git a/search2/indexing/indexerCore.inc.php b/search2/indexing/indexerCore.inc.php index d8ba1b9..a6435c7 100644 --- a/search2/indexing/indexerCore.inc.php +++ b/search2/indexing/indexerCore.inc.php @@ -110,7 +110,7 @@ class QueryResultItem dcv.minor_version, dcv.filename, cou.name as checkoutuser, w.human_name as workflow, ws.human_name as workflowstate, mt.mimetypes as mimetype, md.mime_doc as mimedoc, d.checkedout, mbu.name as modifiedbyuser, d.modified, cbu.name as createdbyuser, ou.name as owneruser, d.immutable, d.status_id, d.created,dcv.storage_path, dtl.name as document_type, - mt.icon_path as mime_icon_path, mt.friendly_name as mime_display, d.oem_no + mt.icon_path as mime_icon_path, mt.friendly_name as mime_display, d.oem_no, dmv.name as title FROM documents d INNER JOIN document_metadata_version dmv ON d.metadata_version_id = dmv.id @@ -151,6 +151,7 @@ class QueryResultItem $this->filename=$result['filename']; $this->filesize = KTUtil::filesizeToString($result['filesize']); $this->folderId = $result['folder_id']; + $this->title = $result['title']; $this->createdBy = $result['createdbyuser']; $this->dateCreated = $result['created']; @@ -510,7 +511,11 @@ abstract class Indexer { $userid=$_SESSION['userID']; if (empty($userid)) $userid=1; - $sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'C' FROM documents WHERE status_id=1"; + + $sql = "DELETE FROM index_files"; + DBUtil::runQuery($sql); + + $sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'A' FROM documents WHERE status_id=1 and id not in (select document_id from index_files)"; DBUtil::runQuery($sql); } @@ -895,7 +900,7 @@ abstract class Indexer Indexer::clearoutDeleted(); - $date = date('Y-m-d H:j:s'); + $date = date('Y-m-d H:i:s'); // identify the indexers that must run // mysql specific limit! $sql = "SELECT @@ -959,18 +964,38 @@ abstract class Indexer $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Indexing docid: %d extension: '%s' mimetype: '%s' extractor: '%s'"), $docId, $extension,$mimeType,$extractorClass), 'debug'); - if (empty($extractorClass)) { - Indexer::unqueueDocument($docId, sprintf(_kt("No extractor for docid: %d"),$docId)); - continue; + /* + + if no extractor is found and we don't need to index discussions, then we can remove the item from the queue. + + */ + if ($indexDiscussion) + { + $indexDocument = false; + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Not indexing docid: %d content because extractor could not be resolve. Still indexing discussion."), $docId), 'info'); + } + else + { + Indexer::unqueueDocument($docId, sprintf(_kt("No extractor for docid: %d"),$docId)); + continue; + } } + else + { + /* - if (!$this->isExtractorEnabled($extractorClass)) - { - $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info'); - continue; - } + If an extractor is available, we must ensure it is enabled. + + */ + + if (!$this->isExtractorEnabled($extractorClass)) + { + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info'); + continue; + } + } if ($this->debug) { @@ -1048,14 +1073,21 @@ abstract class Indexer $title = $document->getName(); if ($indexDiscussion) { - $indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version); - $removeFromQueue = $indexStatus; - if (!$indexStatus) + if (!$this->filterText($targetFile)) { - $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId), 'error'); + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"),$docId), 'error'); } + else + { + $indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version); + $removeFromQueue = $indexStatus; + if (!$indexStatus) + { + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId), 'error'); + } - $extractor->setIndexingStatus($indexStatus); + $extractor->setIndexingStatus($indexStatus); + } } else { @@ -1101,7 +1133,8 @@ abstract class Indexer } else { - $this->indexDiscussion($docId); + $indexStatus = $this->indexDiscussion($docId); + $removeFromQueue = $indexStatus; } if ($removeFromQueue) diff --git a/search2/search/fields/DiscussionTextField.inc.php b/search2/search/fields/DiscussionTextField.inc.php index a7ab6b7..ce5469d 100644 --- a/search2/search/fields/DiscussionTextField.inc.php +++ b/search2/search/fields/DiscussionTextField.inc.php @@ -38,6 +38,8 @@ class DiscussionTextField extends SearchableText { + public $general_op = ExprOp::CONTAINS; + public function __construct() { parent::__construct('Discussion', _kt('Discussion Text')); -- libgit2 0.21.4