Peter M. Groen / knowledgetree

Commit 5ed30c96b97d89a1f8a9a81b53d188d2b8fadce1

Authored by kevin_fourie
1 parent 0f946796

Merged in from DEV trunk... 

KTS-3045
"Search2: Documents are not being indexed in Windows XP for OSS"
Fixed.

Committed By: Conrad Vermeulen
Reviewed By: Megan Watson

KTS-1399
"Searching in discussion threads yields no results"
Fixed.

Committed By: Conrad Vermeulen
Reviewed By: Megan Watson

KTC-177
"RSS feed appears wrong when launched from the 'Actions on this folder' when the folder has the internationalisation string as its name or the name of any of its subfolders."
Fixed. Added a check for IE 6, and only send headers then.

Committed By: Megan Watson
Reviewed By: Conrad Vermeulen

KTS-1399
"Searching in discussion threads yields no results"
Fixed.

Committed By: Conrad Vermeulen
Reviewed By: Megan Watson

KTC-188
"File using Internationalisation string as title and/or filename causes bulk export to break"
Fixed. Added a replace to replace the illegal characters (*,?,etc) in the filenames with dashes.

Committed by: Megan Watson
Reviewed by: Conrad Vermeulen

KTC-383
"License Mechanism modification"
In progress. Changed SMB and Enterprise to Basic, Plus and Premium. Added a check on the tiers instead of the number of licenses.

Committed by: Megan Watson
Reviewed by: Conrad Vermeulen

KTS-3067
"The check on absolute path causes errors with the plugins"
Fixed. Used strtolower on the paths

Committed By: Megan Watson
Reviewed By: Conrad Vermeulen


git-svn-id: https://kt-dms.svn.sourceforge.net/svnroot/kt-dms/STABLE/trunk@8128 c91229c3-7414-0410-bfa2-8a42b809f60b
Inline Side-by-side
Showing 9 changed files with 112 additions and 55 deletions
bin/luceneserver/ktlucene.jar
View file @5ed30c9
No preview for this file type
lib/foldermanagement/compressionArchiveUtil.inc.php
  View file @5ed30c9
... ... @@ -43,6 +43,8 @@ class ZipFolder {
43 43 var $sTmpPath = '';
44 44 var $sZipFileName = '';
45 45 var $sZipFile = '';
  46 + var $sPattern = '';
  47 + var $sFolderPattern = '';
46 48 var $aPaths = array();
47 49 var $aReplaceKeys = array();
48 50 var $aReplaceValues = array();
... ... @@ -55,6 +57,8 @@ class ZipFolder {
55 57 $this->oKTConfig =& KTConfig::getSingleton();
56 58 $this->oStorage =& KTStorageManagerUtil::getSingleton();
57 59  
  60 + $this->sOutputEncoding = $this->oKTConfig->get('export/encoding', 'UTF-8');
  61 +
58 62 $sBasedir = $this->oKTConfig->get("urls/tmpDirectory");
59 63 $sTmpPath = tempnam($sBasedir, 'kt_compress_zip');
60 64  
... ... @@ -65,6 +69,9 @@ class ZipFolder {
65 69 $this->sZipFileName = $sZipFileName;
66 70 $this->aPaths = array();
67 71  
  72 + $this->sPattern = "[\*|\%|\\\|\/|\<|\>|\+|\:|\?|\||\'|\"]";
  73 + $this->sFolderPattern = "[\*|\%|\<|\>|\+|\:|\?|\||\'|\"]";
  74 +
68 75 $aReplace = array(
69 76 "[" => "[[]",
70 77 " " => "[ ]",
... ... @@ -76,7 +83,7 @@ class ZipFolder {
76 83 $this->aReplaceValues = array_values($aReplace);
77 84 }
78 85  
79   - /**
  86 + /**
80 87 * Return the full path
81 88 *
82 89 * @param mixed $oFolderOrDocument May be a Folder or Document
... ... @@ -109,28 +116,30 @@ class ZipFolder {
109 116 }
110 117  
111 118 $sDocPath = $this->getFullFolderPath($oFolder);
  119 + $sDocPath = preg_replace($this->sFolderPattern, '-', $sDocPath);
  120 + $sDocPath = $this->_convertEncoding($sDocPath, true);
  121 +
  122 +
112 123 $sDocName = $oDocument->getFileName();
  124 + $sDocName = preg_replace($this->sPattern, '-', $sDocName);
  125 + $sDocName = $this->_convertEncoding($sDocName, true);
113 126  
114   - $sParentFolder = str_replace('<', '', str_replace('</', '', str_replace('>', '', sprintf('%s/%s', $this->sTmpPath, $sDocPath))));
  127 + $sParentFolder = $this->sTmpPath.'/'.$sDocPath;
115 128 $newDir = $this->sTmpPath;
116   - $sFullPath = str_replace('<', '', str_replace('</', '', str_replace('>', '', $this->_convertEncoding($sDocPath, true))));
117   - foreach (split('/', $sFullPath) as $dirPart) {
  129 +
  130 + $aFullPath = split('/', $sDocPath);
  131 + foreach ($aFullPath as $dirPart) {
118 132 $newDir = sprintf("%s/%s", $newDir, $dirPart);
119 133 if (!file_exists($newDir)) {
120 134 mkdir($newDir, 0700);
121 135 }
122 136 }
123 137  
124   - $sOrigFile = str_replace('<', '', str_replace('</', '', str_replace('>', '', $this->oStorage->temporaryFile($oDocument))));
125   - $sFilename = sprintf("%s/%s", $sParentFolder, str_replace('<', '', str_replace('</', '', str_replace('>', '', $sDocName))));
126   - $sFilename = $this->_convertEncoding($sFilename, true);
  138 + $sOrigFile = $this->oStorage->temporaryFile($oDocument);
  139 + $sFilename = $sParentFolder.'/'.$sDocName;
127 140 copy($sOrigFile, $sFilename);
128 141  
129   - $sPath = str_replace('<', '', str_replace('</', '', str_replace('>', '', sprintf("%s/%s", $sDocPath, $sDocName))));
130   - $sPath = str_replace($this->aReplaceKeys, $this->aReplaceValues, $sPath);
131   - $sPath = $this->_convertEncoding($sPath, true);
132   -
133   - $this->aPaths[] = $sPath;
  142 + $this->aPaths[] = $sDocPath.'/'.$sDocName;
134 143 return true;
135 144 }
136 145  
... ... @@ -139,21 +148,20 @@ class ZipFolder {
139 148 */
140 149 function addFolderToZip($oFolder) {
141 150 $sFolderPath = $this->getFullFolderPath($oFolder) .'/';
142   - $sParentFolder = str_replace('<', '', str_replace('</', '', str_replace('>', '', sprintf('%s/%s', $this->sTmpPath, $sFolderPath))));
  151 + $sFolderPath = preg_replace($this->sFolderPattern, '-', $sFolderPath);
  152 + $sFolderPath = $this->_convertEncoding($sFolderPath, true);
  153 +
143 154 $newDir = $this->sTmpPath;
144   - $sFullPath = str_replace('<', '', str_replace('</', '', str_replace('>', '', $this->_convertEncoding($sFolderPath, true))));
145   - foreach (split('/', $sFullPath) as $dirPart) {
  155 +
  156 + $aFullPath = split('/', $sFolderPath);
  157 + foreach ($aFullPath as $dirPart) {
146 158 $newDir = sprintf("%s/%s", $newDir, $dirPart);
147 159 if (!file_exists($newDir)) {
148 160 mkdir($newDir, 0700);
149 161 }
150 162 }
151 163  
152   - $sPath = str_replace('<', '', str_replace('</', '', str_replace('>', '', sprintf("%s", $sFolderPath))));
153   - $sPath = str_replace($this->aReplaceKeys, $this->aReplaceValues, $sPath);
154   - $sPath = $this->_convertEncoding($sPath, true);
155   -
156   - $this->aPaths[] = $sPath;
  164 + $this->aPaths[] = $sFolderPath;
157 165 return true;
158 166 }
159 167  
... ... @@ -163,10 +171,14 @@ class ZipFolder {
163 171 function createZipFile($bEchoStatus = FALSE) {
164 172 if(empty($this->aPaths)){
165 173 return PEAR::raiseError(_kt("No folders or documents found to compress"));
166   - //$this->addErrorMessage(_kt("No folders or documents found to compress"));
167   - //return false;
168 174 }
169 175  
  176 + // Set environment language to output character encoding
  177 + $loc = $this->sOutputEncoding;
  178 + putenv("LANG=$loc");
  179 + putenv("LANGUAGE=$loc");
  180 + $loc = setlocale(LC_ALL, $loc);
  181 +
170 182 $sManifest = sprintf("%s/%s", $this->sTmpPath, "MANIFEST");
171 183 file_put_contents($sManifest, join("\n", $this->aPaths));
172 184 $sZipFile = sprintf("%s/%s.zip", $this->sTmpPath, $this->sZipFileName);
... ... @@ -218,6 +230,7 @@ class ZipFolder {
218 230 if(!(isset($exportCode) && !empty($exportCode))) {
219 231 $exportCode = KTUtil::arrayGet($_SESSION['zipcompression'], 'exportcode');
220 232 }
  233 +
221 234 $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode);
222 235  
223 236 if(!empty($aData)){
... ... @@ -232,7 +245,7 @@ class ZipFolder {
232 245 return PEAR::raiseError(_kt('The ZIP file can only be downloaded once - if you cancel the download, you will need to reload the page.'));
233 246 }
234 247  
235   - header("Content-Type: application/zip");
  248 + header("Content-Type: application/zip; charset=utf-8");
236 249 header("Content-Length: ". filesize($sZipFile));
237 250 header("Content-Disposition: attachment; filename=\"" . $this->sZipFileName . ".zip" . "\"");
238 251 header("Expires: Mon, 26 Jul 1997 05:00:00 GMT");
... ...
lib/plugins/pluginutil.inc.php
  View file @5ed30c9
... ... @@ -412,7 +412,8 @@ class KTPluginUtil {
412 412 }
413 413 }
414 414  
415   - foreach (KTPluginEntity::getList() as $oPluginEntity) {
  415 + $aPluginList = KTPluginEntity::getList();
  416 + foreach ($aPluginList as $oPluginEntity) {
416 417 $sPath = $oPluginEntity->getPath();
417 418 if (!KTUtil::isAbsolutePath($sPath)) {
418 419 $sPath = sprintf("%s/%s", KT_DIR, $sPath);
... ...
lib/util/ktutil.inc
  View file @5ed30c9
... ... @@ -786,7 +786,7 @@ class KTUtil {
786 786 $sPath = str_replace('\\', '/', $sPath);
787 787 $sReal = str_replace('\\', '/', realpath($sPath));
788 788  
789   - return ($sReal == $sPath);
  789 + return (strtolower($sReal) == strtolower($sPath));
790 790  
791 791 if (substr($sPath, 0, 1) == '/') {
792 792 return true;
... ...
... ... @@ -74,10 +74,16 @@ if (!validateUser($_SERVER[&#39;PHP_AUTH_USER&#39;], $_SERVER[&#39;PHP_AUTH_PW&#39;])) {
74 74 $user = DBAuthenticator::getUser($_SERVER['PHP_AUTH_USER'], array('id'=>'id',));
75 75 $id = $user[$_SERVER['PHP_AUTH_USER']]['id'];
76 76  
77   - header('Content-Type: application/rss+xml; charset=utf-8;');
78   - header('Content-Disposition: inline; filename="rss.xml"');
79   - header('Expires: Mon, 26 Jul 1997 05:00:00 GMT');
80   - header('Last-Modified: ' . gmdate('D, d M Y H:i:s') . ' GMT');
  77 + if(OS_WINDOWS){
  78 + $sReferrer = $_SERVER['HTTP_USER_AGENT'];
  79 + // Check if this is IE 6
  80 + if(strstr($sReferrer, 'MSIE 6.0')){
  81 + header('Content-Type: application/rss+xml; charset=utf-8;');
  82 + header('Content-Disposition: inline; filename="rss.xml"');
  83 + header('Expires: Mon, 26 Jul 1997 05:00:00 GMT');
  84 + header('Last-Modified: ' . gmdate('D, d M Y H:i:s') . ' GMT');
  85 + }
  86 + }
81 87  
82 88 if(KTUtil::arrayGet($_REQUEST, 'docId')){ // if a docId parameter is passed
83 89 // get document id from http request object
... ...
search2/indexing/extractorCore.inc.php
  View file @5ed30c9
... ... @@ -370,7 +370,9 @@ abstract class ExternalDocumentExtractor extends DocumentExtractor
370 370 $script_name = $script_prefix . '.bat';
371 371  
372 372 $script = "rem This is an auto generated file. \n";
373   - $script .= $cmd . ' 2>>"' . $script_out . '" >>"' . $script_out . "\"\r\n";
  373 + $script .= $cmd . ' 2>"' . $script_out . "\"\r\n";
  374 + $script .= "set er=%ERRORLEVEL%\r\n";
  375 + $script .= "exit /B %er%\r\n";
374 376 }
375 377 else
376 378 {
... ... @@ -392,9 +394,8 @@ abstract class ExternalDocumentExtractor extends DocumentExtractor
392 394 // execute the script file
393 395 if (OS_WINDOWS)
394 396 {
395   - $script_name = "\"$script_name\"";
396   - $WshShell = new COM("WScript.Shell");
397   - $res = $WshShell->Run($script_name, 0, true);
  397 + $res = KTUtil::pexec("\"$script_name\"");
  398 + $res = $res['ret'];
398 399 }
399 400 else
400 401 {
... ...
search2/indexing/extractors/OOTextExtractor.inc.php
  View file @5ed30c9
... ... @@ -88,16 +88,17 @@ class OOTextExtractor extends ExternalDocumentExtractor
88 88  
89 89 protected function getCommandLine()
90 90 {
91   - //$sourcefile = escapeshellcmd($this->sourcefile);
92   - $sourcefile = ($this->sourcefile);
  91 + $sourcefile = $this->sourcefile;
  92 +
93 93 unlink($this->targetfile);
94 94 $this->targetfile .= '.' . $this->targetExtension;
95   - //$targetfile = escapeshellcmd($this->targetfile);
96   - $targetfile = ($this->targetfile);
  95 + $targetfile = $this->targetfile;
97 96  
98   - $escape = OS_WINDOWS?'"':'\'';
  97 + $escape = '"';
99 98  
100 99 $cmdline = "{$escape}{$this->python}{$escape} {$escape}{$this->documentConverter}{$escape} {$escape}{$sourcefile}{$escape} {$escape}{$targetfile}{$escape} {$this->ooHost} {$this->ooPort}";
  100 + $cmdline = str_replace('\\','/',$cmdline);
  101 +
101 102 return $cmdline;
102 103 }
103 104  
... ...
search2/indexing/indexerCore.inc.php
  View file @5ed30c9
... ... @@ -110,7 +110,7 @@ class QueryResultItem
110 110 dcv.minor_version, dcv.filename, cou.name as checkoutuser, w.human_name as workflow, ws.human_name as workflowstate,
111 111 mt.mimetypes as mimetype, md.mime_doc as mimedoc, d.checkedout, mbu.name as modifiedbyuser, d.modified,
112 112 cbu.name as createdbyuser, ou.name as owneruser, d.immutable, d.status_id, d.created,dcv.storage_path, dtl.name as document_type,
113   - mt.icon_path as mime_icon_path, mt.friendly_name as mime_display, d.oem_no
  113 + mt.icon_path as mime_icon_path, mt.friendly_name as mime_display, d.oem_no, dmv.name as title
114 114 FROM
115 115 documents d
116 116 INNER JOIN document_metadata_version dmv ON d.metadata_version_id = dmv.id
... ... @@ -151,6 +151,7 @@ class QueryResultItem
151 151 $this->filename=$result['filename'];
152 152 $this->filesize = KTUtil::filesizeToString($result['filesize']);
153 153 $this->folderId = $result['folder_id'];
  154 + $this->title = $result['title'];
154 155  
155 156 $this->createdBy = $result['createdbyuser'];
156 157 $this->dateCreated = $result['created'];
... ... @@ -510,7 +511,11 @@ abstract class Indexer
510 511 {
511 512 $userid=$_SESSION['userID'];
512 513 if (empty($userid)) $userid=1;
513   - $sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'C' FROM documents WHERE status_id=1";
  514 +
  515 + $sql = "DELETE FROM index_files";
  516 + DBUtil::runQuery($sql);
  517 +
  518 + $sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'A' FROM documents WHERE status_id=1 and id not in (select document_id from index_files)";
514 519 DBUtil::runQuery($sql);
515 520 }
516 521  
... ... @@ -895,7 +900,7 @@ abstract class Indexer
895 900  
896 901 Indexer::clearoutDeleted();
897 902  
898   - $date = date('Y-m-d H:j:s');
  903 + $date = date('Y-m-d H:i:s');
899 904 // identify the indexers that must run
900 905 // mysql specific limit!
901 906 $sql = "SELECT
... ... @@ -959,18 +964,38 @@ abstract class Indexer
959 964  
960 965 $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Indexing docid: %d extension: '%s' mimetype: '%s' extractor: '%s'"), $docId, $extension,$mimeType,$extractorClass), 'debug');
961 966  
962   -
963 967 if (empty($extractorClass))
964 968 {
965   - Indexer::unqueueDocument($docId, sprintf(_kt("No extractor for docid: %d"),$docId));
966   - continue;
  969 + /*
  970 +
  971 + if no extractor is found and we don't need to index discussions, then we can remove the item from the queue.
  972 +
  973 + */
  974 + if ($indexDiscussion)
  975 + {
  976 + $indexDocument = false;
  977 + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Not indexing docid: %d content because extractor could not be resolve. Still indexing discussion."), $docId), 'info');
  978 + }
  979 + else
  980 + {
  981 + Indexer::unqueueDocument($docId, sprintf(_kt("No extractor for docid: %d"),$docId));
  982 + continue;
  983 + }
967 984 }
  985 + else
  986 + {
  987 + /*
968 988  
969   - if (!$this->isExtractorEnabled($extractorClass))
970   - {
971   - $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info');
972   - continue;
973   - }
  989 + If an extractor is available, we must ensure it is enabled.
  990 +
  991 + */
  992 +
  993 + if (!$this->isExtractorEnabled($extractorClass))
  994 + {
  995 + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info');
  996 + continue;
  997 + }
  998 + }
974 999  
975 1000 if ($this->debug)
976 1001 {
... ... @@ -1048,14 +1073,21 @@ abstract class Indexer
1048 1073 $title = $document->getName();
1049 1074 if ($indexDiscussion)
1050 1075 {
1051   - $indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version);
1052   - $removeFromQueue = $indexStatus;
1053   - if (!$indexStatus)
  1076 + if (!$this->filterText($targetFile))
1054 1077 {
1055   - $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId), 'error');
  1078 + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"),$docId), 'error');
1056 1079 }
  1080 + else
  1081 + {
  1082 + $indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version);
  1083 + $removeFromQueue = $indexStatus;
  1084 + if (!$indexStatus)
  1085 + {
  1086 + $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId), 'error');
  1087 + }
1057 1088  
1058   - $extractor->setIndexingStatus($indexStatus);
  1089 + $extractor->setIndexingStatus($indexStatus);
  1090 + }
1059 1091 }
1060 1092 else
1061 1093 {
... ... @@ -1101,7 +1133,8 @@ abstract class Indexer
1101 1133 }
1102 1134 else
1103 1135 {
1104   - $this->indexDiscussion($docId);
  1136 + $indexStatus = $this->indexDiscussion($docId);
  1137 + $removeFromQueue = $indexStatus;
1105 1138 }
1106 1139  
1107 1140 if ($removeFromQueue)
... ...
search2/search/fields/DiscussionTextField.inc.php
  View file @5ed30c9
... ... @@ -38,6 +38,8 @@
38 38  
39 39 class DiscussionTextField extends SearchableText
40 40 {
  41 + public $general_op = ExprOp::CONTAINS;
  42 +
41 43 public function __construct()
42 44 {
43 45 parent::__construct('Discussion', _kt('Discussion Text'));
... ...