Merged in from DEV trunk...

KTS-3045 "Search2: Documents are not being indexed in Windows XP for OSS" Fixed. Committed By: Conrad Vermeulen Reviewed By: Megan Watson KTS-1399 "Searching in discussion threads yields no results" Fixed. Committed By: Conrad Vermeulen Reviewed By: Megan Watson KTC-177 "RSS feed appears wrong when launched from the 'Actions on this folder' when the folder has the internationalisation string as its name or the name of any of its subfolders." Fixed. Added a check for IE 6, and only send headers then. Committed By: Megan Watson Reviewed By: Conrad Vermeulen KTS-1399 "Searching in discussion threads yields no results" Fixed. Committed By: Conrad Vermeulen Reviewed By: Megan Watson KTC-188 "File using Internationalisation string as title and/or filename causes bulk export to break" Fixed. Added a replace to replace the illegal characters (*,?,etc) in the filenames with dashes. Committed by: Megan Watson Reviewed by: Conrad Vermeulen KTC-383 "License Mechanism modification" In progress. Changed SMB and Enterprise to Basic, Plus and Premium. Added a check on the tiers instead of the number of licenses. Committed by: Megan Watson Reviewed by: Conrad Vermeulen KTS-3067 "The check on absolute path causes errors with the plugins" Fixed. Used strtolower on the paths Committed By: Megan Watson Reviewed By: Conrad Vermeulen git-svn-id: https://kt-dms.svn.sourceforge.net/svnroot/kt-dms/STABLE/trunk@8128 c91229c3-7414-0410-bfa2-8a42b809f60b

Merged in from DEV trunk...
KTS-3045 "Search2: Documents are not being indexed in Windows XP for OSS" Fixed. Committed By: Conrad Vermeulen Reviewed By: Megan Watson KTS-1399 "Searching in discussion threads yields no results" Fixed. Committed By: Conrad Vermeulen Reviewed By: Megan Watson KTC-177 "RSS feed appears wrong when launched from the 'Actions on this folder' when the folder has the internationalisation string as its name or the name of any of its subfolders." Fixed. Added a check for IE 6, and only send headers then. Committed By: Megan Watson Reviewed By: Conrad Vermeulen KTS-1399 "Searching in discussion threads yields no results" Fixed. Committed By: Conrad Vermeulen Reviewed By: Megan Watson KTC-188 "File using Internationalisation string as title and/or filename causes bulk export to break" Fixed. Added a replace to replace the illegal characters (*,?,etc) in the filenames with dashes. Committed by: Megan Watson Reviewed by: Conrad Vermeulen KTC-383 "License Mechanism modification" In progress. Changed SMB and Enterprise to Basic, Plus and Premium. Added a check on the tiers instead of the number of licenses. Committed by: Megan Watson Reviewed by: Conrad Vermeulen KTS-3067 "The check on absolute path causes errors with the plugins" Fixed. Used strtolower on the paths Committed By: Megan Watson Reviewed By: Conrad Vermeulen git-svn-id: https://kt-dms.svn.sourceforge.net/svnroot/kt-dms/STABLE/trunk@8128 c91229c3-7414-0410-bfa2-8a42b809f60b
kevin_fourie
1 parent 0f946796
Showing 9 changed files with 112 additions and 55 deletions
bin/luceneserver/ktlucene.jar
lib/foldermanagement/compressionArchiveUtil.inc.php
lib/plugins/pluginutil.inc.php
lib/util/ktutil.inc
rss.php
search2/indexing/extractorCore.inc.php
search2/indexing/extractors/OOTextExtractor.inc.php
search2/indexing/indexerCore.inc.php
search2/search/fields/DiscussionTextField.inc.php
@@ -43,6 +43,8 @@ class ZipFolder {
     var $sTmpPath = '';
     var $sZipFileName = '';
     var $sZipFile = '';
+    var $sPattern = '';
+    var $sFolderPattern = '';
     var $aPaths = array();
     var $aReplaceKeys = array();
     var $aReplaceValues = array();
@@ -55,6 +57,8 @@ class ZipFolder {
         $this->oKTConfig =& KTConfig::getSingleton();
         $this->oStorage =& KTStorageManagerUtil::getSingleton();
+        $this->sOutputEncoding = $this->oKTConfig->get('export/encoding', 'UTF-8');
+
         $sBasedir = $this->oKTConfig->get("urls/tmpDirectory");
         $sTmpPath = tempnam($sBasedir, 'kt_compress_zip');
@@ -65,6 +69,9 @@ class ZipFolder {
         $this->sZipFileName = $sZipFileName;
         $this->aPaths = array();
+        $this->sPattern = "[\*|\%|\\\|\/|\<|\>|\+|\:|\?|\||\'|\"]";
+        $this->sFolderPattern = "[\*|\%|\<|\>|\+|\:|\?|\||\'|\"]";
+
         $aReplace = array(
             "[" => "[[]",
             " " => "[ ]",
@@ -76,7 +83,7 @@ class ZipFolder {
         $this->aReplaceValues = array_values($aReplace);
     }
- /**
+    /**
      * Return the full path
      *
      * @param mixed $oFolderOrDocument May be a Folder or Document
@@ -109,28 +116,30 @@ class ZipFolder {
         }
         $sDocPath = $this->getFullFolderPath($oFolder);
+        $sDocPath = preg_replace($this->sFolderPattern, '-', $sDocPath);
+        $sDocPath = $this->_convertEncoding($sDocPath, true);
+
+
         $sDocName = $oDocument->getFileName();
+        $sDocName = preg_replace($this->sPattern, '-', $sDocName);
+        $sDocName = $this->_convertEncoding($sDocName, true);
-        $sParentFolder = str_replace('<', '', str_replace('</', '', str_replace('>', '', sprintf('%s/%s', $this->sTmpPath, $sDocPath))));
+        $sParentFolder = $this->sTmpPath.'/'.$sDocPath;
         $newDir = $this->sTmpPath;
-        $sFullPath = str_replace('<', '', str_replace('</', '', str_replace('>', '', $this->_convertEncoding($sDocPath, true))));
-        foreach (split('/', $sFullPath) as $dirPart) {
+
+        $aFullPath = split('/', $sDocPath);
+        foreach ($aFullPath as $dirPart) {
             $newDir = sprintf("%s/%s", $newDir, $dirPart);
             if (!file_exists($newDir)) {
                 mkdir($newDir, 0700);
             }
         }
-        $sOrigFile = str_replace('<', '', str_replace('</', '', str_replace('>', '', $this->oStorage->temporaryFile($oDocument))));
-        $sFilename = sprintf("%s/%s", $sParentFolder, str_replace('<', '', str_replace('</', '', str_replace('>', '', $sDocName))));
-        $sFilename = $this->_convertEncoding($sFilename, true);
+        $sOrigFile = $this->oStorage->temporaryFile($oDocument);
+        $sFilename = $sParentFolder.'/'.$sDocName;
         copy($sOrigFile, $sFilename);
-        $sPath = str_replace('<', '', str_replace('</', '', str_replace('>', '', sprintf("%s/%s", $sDocPath, $sDocName))));
-        $sPath = str_replace($this->aReplaceKeys, $this->aReplaceValues, $sPath);
-        $sPath = $this->_convertEncoding($sPath, true);
-
-        $this->aPaths[] = $sPath;
+        $this->aPaths[] = $sDocPath.'/'.$sDocName;
         return true;
     }
@@ -139,21 +148,20 @@ class ZipFolder {
     */
     function addFolderToZip($oFolder) {
         $sFolderPath = $this->getFullFolderPath($oFolder) .'/';
-        $sParentFolder = str_replace('<', '', str_replace('</', '', str_replace('>', '', sprintf('%s/%s', $this->sTmpPath, $sFolderPath))));
+        $sFolderPath = preg_replace($this->sFolderPattern, '-', $sFolderPath);
+        $sFolderPath = $this->_convertEncoding($sFolderPath, true);
+
         $newDir = $this->sTmpPath;
-        $sFullPath = str_replace('<', '', str_replace('</', '', str_replace('>', '', $this->_convertEncoding($sFolderPath, true))));
-        foreach (split('/', $sFullPath) as $dirPart) {
+
+        $aFullPath = split('/', $sFolderPath);
+        foreach ($aFullPath as $dirPart) {
             $newDir = sprintf("%s/%s", $newDir, $dirPart);
             if (!file_exists($newDir)) {
                 mkdir($newDir, 0700);
             }
         }
-        $sPath = str_replace('<', '', str_replace('</', '', str_replace('>', '', sprintf("%s", $sFolderPath))));
-        $sPath = str_replace($this->aReplaceKeys, $this->aReplaceValues, $sPath);
-        $sPath = $this->_convertEncoding($sPath, true);
-
-        $this->aPaths[] = $sPath;
+        $this->aPaths[] = $sFolderPath;
         return true;
     }
@@ -163,10 +171,14 @@ class ZipFolder {
     function createZipFile($bEchoStatus = FALSE) {
         if(empty($this->aPaths)){
             return PEAR::raiseError(_kt("No folders or documents found to compress"));
-            //$this->addErrorMessage(_kt("No folders or documents found to compress"));
-            //return false;
         }
+        // Set environment language to output character encoding
+        $loc = $this->sOutputEncoding;
+        putenv("LANG=$loc");
+        putenv("LANGUAGE=$loc");
+        $loc = setlocale(LC_ALL, $loc);
+
         $sManifest = sprintf("%s/%s", $this->sTmpPath, "MANIFEST");
         file_put_contents($sManifest, join("\n", $this->aPaths));
         $sZipFile = sprintf("%s/%s.zip", $this->sTmpPath, $this->sZipFileName);
@@ -218,6 +230,7 @@ class ZipFolder {
         if(!(isset($exportCode) && !empty($exportCode))) {
             $exportCode = KTUtil::arrayGet($_SESSION['zipcompression'], 'exportcode');
         }
+
         $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode);
         if(!empty($aData)){
@@ -232,7 +245,7 @@ class ZipFolder {
             return PEAR::raiseError(_kt('The ZIP file can only be downloaded once - if you cancel the download, you will need to reload the page.'));
         }
-		header("Content-Type: application/zip");
+        header("Content-Type: application/zip; charset=utf-8");
         header("Content-Length: ". filesize($sZipFile));
         header("Content-Disposition: attachment; filename=\"" . $this->sZipFileName . ".zip" . "\"");
         header("Expires: Mon, 26 Jul 1997 05:00:00 GMT");
@@ -412,7 +412,8 @@ class KTPluginUtil {
             }
         }
-        foreach (KTPluginEntity::getList() as $oPluginEntity) {
+        $aPluginList = KTPluginEntity::getList();
+        foreach ($aPluginList as $oPluginEntity) {
             $sPath = $oPluginEntity->getPath();
             if (!KTUtil::isAbsolutePath($sPath)) {
                 $sPath = sprintf("%s/%s", KT_DIR, $sPath);
@@ -786,7 +786,7 @@ class KTUtil {
         $sPath = str_replace('\\', '/', $sPath);
         $sReal = str_replace('\\', '/', realpath($sPath));
-        return ($sReal == $sPath);
+        return (strtolower($sReal) == strtolower($sPath));
         if (substr($sPath, 0, 1) == '/') {
             return true;
@@ -74,10 +74,16 @@ if (!validateUser($_SERVER[&#39;PHP_AUTH_USER&#39;], $_SERVER[&#39;PHP_AUTH_PW&#39;])) {
     $user = DBAuthenticator::getUser($_SERVER['PHP_AUTH_USER'], array('id'=>'id',));
     $id =  $user[$_SERVER['PHP_AUTH_USER']]['id'];
-    header('Content-Type: application/rss+xml; charset=utf-8;');
-    header('Content-Disposition: inline; filename="rss.xml"');
-    header('Expires: Mon, 26 Jul 1997 05:00:00 GMT');
-    header('Last-Modified: ' . gmdate('D, d M Y H:i:s') . ' GMT');
+    if(OS_WINDOWS){
+    	$sReferrer = $_SERVER['HTTP_USER_AGENT'];
+    	// Check if this is IE 6
+    	if(strstr($sReferrer, 'MSIE 6.0')){
+    		header('Content-Type: application/rss+xml; charset=utf-8;');
+		    header('Content-Disposition: inline; filename="rss.xml"');
+		    header('Expires: Mon, 26 Jul 1997 05:00:00 GMT');
+		    header('Last-Modified: ' . gmdate('D, d M Y H:i:s') . ' GMT');
+    	}
+    }
     if(KTUtil::arrayGet($_REQUEST, 'docId')){ // if a docId parameter is passed
         // get document id from http request object
@@ -370,7 +370,9 @@ abstract class ExternalDocumentExtractor extends DocumentExtractor
 			$script_name = $script_prefix . '.bat';
 			$script = "rem This is an auto generated file. \n";
-			$script .= $cmd . ' 2>>"' . $script_out . '" >>"' . $script_out . "\"\r\n";
+			$script .= $cmd . ' 2>"' . $script_out . "\"\r\n";
+			$script .= "set er=%ERRORLEVEL%\r\n";
+			$script .= "exit /B %er%\r\n";
 		}
 		else
 		{
@@ -392,9 +394,8 @@ abstract class ExternalDocumentExtractor extends DocumentExtractor
 		// execute the script file
 		if (OS_WINDOWS)
 		{
-			$script_name = "\"$script_name\"";
-			$WshShell = new COM("WScript.Shell");
-			$res = $WshShell->Run($script_name, 0, true);
+			$res = KTUtil::pexec("\"$script_name\"");
+			$res = $res['ret'];
 		}
 		else
 		{
@@ -88,16 +88,17 @@ class OOTextExtractor extends ExternalDocumentExtractor
 	protected function getCommandLine()
 	{
-		//$sourcefile = escapeshellcmd($this->sourcefile);
-		$sourcefile = ($this->sourcefile);
+		$sourcefile = $this->sourcefile;
+
 		unlink($this->targetfile);
 		$this->targetfile .= '.' . $this->targetExtension;
-		//$targetfile = escapeshellcmd($this->targetfile);
-		$targetfile =  ($this->targetfile);
+		$targetfile = $this->targetfile;
-		$escape = OS_WINDOWS?'"':'\'';
+		$escape = '"';
 		$cmdline = "{$escape}{$this->python}{$escape} {$escape}{$this->documentConverter}{$escape} {$escape}{$sourcefile}{$escape} {$escape}{$targetfile}{$escape} {$this->ooHost} {$this->ooPort}";
+		$cmdline = str_replace('\\','/',$cmdline);
+
 		return $cmdline;
 	}
@@ -110,7 +110,7 @@ class QueryResultItem
 					dcv.minor_version, dcv.filename, cou.name as checkoutuser, w.human_name as workflow, ws.human_name as workflowstate,
 					mt.mimetypes as mimetype, md.mime_doc as mimedoc, d.checkedout, mbu.name as modifiedbyuser, d.modified,
 					cbu.name as createdbyuser, ou.name as owneruser, d.immutable, d.status_id, d.created,dcv.storage_path, dtl.name as document_type,
-					mt.icon_path as mime_icon_path, mt.friendly_name as mime_display, d.oem_no
+					mt.icon_path as mime_icon_path, mt.friendly_name as mime_display, d.oem_no, dmv.name as title
 				FROM
 					documents d
 					INNER JOIN document_metadata_version dmv ON d.metadata_version_id = dmv.id
@@ -151,6 +151,7 @@ class QueryResultItem
 		$this->filename=$result['filename'];
 		$this->filesize = KTUtil::filesizeToString($result['filesize']);
 		$this->folderId = $result['folder_id'];
+		$this->title = $result['title'];
 		$this->createdBy = $result['createdbyuser'];
 		$this->dateCreated = $result['created'];
@@ -510,7 +511,11 @@ abstract class Indexer
     {
     	 $userid=$_SESSION['userID'];
     	 if (empty($userid)) $userid=1;
-    	$sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'C' FROM documents WHERE status_id=1";
+
+    	$sql = "DELETE FROM index_files";
+    	DBUtil::runQuery($sql);
+
+    	$sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'A' FROM documents WHERE status_id=1 and id not in (select document_id from index_files)";
     	DBUtil::runQuery($sql);
     }
@@ -895,7 +900,7 @@ abstract class Indexer
     	Indexer::clearoutDeleted();
-    	$date = date('Y-m-d H:j:s');
+    	$date = date('Y-m-d H:i:s');
     	// identify the indexers that must run
         // mysql specific limit!
         $sql = "SELECT
@@ -959,18 +964,38 @@ abstract class Indexer
         	$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Indexing docid: %d extension: '%s' mimetype: '%s' extractor: '%s'"), $docId, $extension,$mimeType,$extractorClass), 'debug');
-
         	if (empty($extractorClass))
         	{
-        		Indexer::unqueueDocument($docId, sprintf(_kt("No extractor for docid: %d"),$docId));
-        		continue;
+        		/*
+
+        		if no extractor is found and we don't need to index discussions, then we can remove the item from the queue.
+
+        		*/
+        		if ($indexDiscussion)
+        		{
+        			$indexDocument = false;
+        			$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Not indexing docid: %d content because extractor could not be resolve. Still indexing discussion."), $docId), 'info');
+        		}
+        		else
+        		{
+        			Indexer::unqueueDocument($docId, sprintf(_kt("No extractor for docid: %d"),$docId));
+        			continue;
+        		}
         	}
+        	else
+        	{
+        		/*
-        	if (!$this->isExtractorEnabled($extractorClass))
-			{
-				$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info');
-				continue;
-			}
+        		If an extractor is available, we must ensure it is enabled.
+
+        		 */
+
+	        	if (!$this->isExtractorEnabled($extractorClass))
+				{
+					$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info');
+					continue;
+				}
+        	}
         	if ($this->debug)
         	{
@@ -1048,14 +1073,21 @@ abstract class Indexer
 					$title = $document->getName();
         			if ($indexDiscussion)
         			{
-        				$indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version);
-        				$removeFromQueue = $indexStatus;
-        				if (!$indexStatus)
+        				if (!$this->filterText($targetFile))
         				{
-        					$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId), 'error');
+        					$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"),$docId), 'error');
         				}
+						else
+						{
+	        				$indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version);
+    	    				$removeFromQueue = $indexStatus;
+        					if (!$indexStatus)
+        					{
+        						$this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId), 'error');
+        					}
-        				$extractor->setIndexingStatus($indexStatus);
+        					$extractor->setIndexingStatus($indexStatus);
+						}
         			}
         			else
         			{
@@ -1101,7 +1133,8 @@ abstract class Indexer
         	}
         	else
         	{
-				$this->indexDiscussion($docId);
+				$indexStatus = $this->indexDiscussion($docId);
+				$removeFromQueue = $indexStatus;
         	}
         	if ($removeFromQueue)
@@ -38,6 +38,8 @@
 class DiscussionTextField extends SearchableText
 {
+	public $general_op = ExprOp::CONTAINS;
+
     public function __construct()
     {
         parent::__construct('Discussion', _kt('Discussion Text'));