Merged in from DEV trunk...

KTS-673 "The search algorithm needs some work" Updated. Integrated with DocumentConverter.py Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-2547 "Test RTF Indexer" Implemented. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-673 "The search algorithm needs some work" Updated. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-673 "The search algorithm needs some work" Updated. MailMime is not tested... will implement again when have more time. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-673 "The search algorithm needs some work" Updated. Sometimes the extractor may change the target filename. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-1753 " Implement Disk Usage Plugin" Updated. Licensing. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-673 "The search algorithm needs some work" Updated. Search requires php, python and java. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie git-svn-id: https://kt-dms.svn.sourceforge.net/svnroot/kt-dms/STABLE/trunk@7477 c91229c3-7414-0410-bfa2-8a42b809f60b

Merged in from DEV trunk...
KTS-673 "The search algorithm needs some work" Updated. Integrated with DocumentConverter.py Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-2547 "Test RTF Indexer" Implemented. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-673 "The search algorithm needs some work" Updated. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-673 "The search algorithm needs some work" Updated. MailMime is not tested... will implement again when have more time. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-673 "The search algorithm needs some work" Updated. Sometimes the extractor may change the target filename. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-1753 " Implement Disk Usage Plugin" Updated. Licensing. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie KTS-673 "The search algorithm needs some work" Updated. Search requires php, python and java. Committed By: Conrad Vermeulen Reviewed By: Kevin Fourie git-svn-id: https://kt-dms.svn.sourceforge.net/svnroot/kt-dms/STABLE/trunk@7477 c91229c3-7414-0410-bfa2-8a42b809f60b
kevin_fourie
1 parent 319ec2a8
Showing 13 changed files with 269 additions and 84 deletions
config/config.ini
plugins/housekeeper/DiskUsageDashlet.inc.php
plugins/housekeeper/FolderUsageDashlet.inc.php
plugins/housekeeper/HouseKeeperDispatcher.php
plugins/housekeeper/HouseKeeperPlugin.php
search2/indexing/extractorCore.inc.php
search2/indexing/extractors/OOPresentationExtractor.inc.php
search2/indexing/extractors/OOSpreadsheetExtractor.inc.php
search2/indexing/extractors/OOTextExtractor.inc.php
search2/indexing/extractors/PDFExtractor.inc.php
search2/indexing/extractors/MailMimeExtractor.inc.php → search2/indexing/extractors/RTFExtractor.inc.php
search2/indexing/extractors/XMLExtractor.inc.php
search2/indexing/indexerCore.inc.php
@@ -242,7 +242,10 @@ pdftotext = pdftotext
 catppt = catppt
 pstotext = pstotext
 catdoc = catdoc
-antiword = antiword.exe
+antiword = antiword
+python = python
+java = java
+php = php
 [search]
 ; The number of results per page
 <?php
 /**
+ * $Id
  *
- * Copyright (c) 2007 Jam Warehouse http://www.jamwarehouse.com
+ * The contents of this file are subject to the KnowledgeTree Public
+ * License Version 1.1.2 ("License"); You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.knowledgetree.com/KPL
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; using version 2 of the License.
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
+ * See the License for the specific language governing rights and
+ * limitations under the License.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+ * All copies of the Covered Code must include on each user interface screen:
+ *    (i) the "Powered by KnowledgeTree" logo and
+ *    (ii) the KnowledgeTree copyright notice
+ * in the same form as they appear in the distribution.  See the License for
+ * requirements.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * The Original Code is: KnowledgeTree Open Source
  *
- * -------------------------------------------------------------------------
- *
- * You can contact the copyright owner regarding licensing via the contact
- * details that can be found on the KnowledgeTree web site:
- *
- *         http://www.knowledgetree.com/
+ * The Initial Developer of the Original Code is The Jam Warehouse Software
+ * (Pty) Ltd, trading as KnowledgeTree.
+ * Portions created by The Jam Warehouse Software (Pty) Ltd are Copyright
+ * (C) 2007 The Jam Warehouse Software (Pty) Ltd;
+ * All Rights Reserved.
+ * Contributor( s): ______________________________________
  */
 class DiskUsageDashlet extends KTBaseDashlet
@@ -34,7 +38,7 @@ class DiskUsageDashlet extends KTBaseDashlet
 	function DiskUsageDashlet()
 	{
-		$this->sTitle = _kt('Disk Usage');
+		$this->sTitle = _kt('Storage Utilization');
 		$this->sClass = "ktInfo";
 	}
 <?php
 /**
+ * $Id
  *
- * Copyright (c) 2007 Jam Warehouse http://www.jamwarehouse.com
+ * The contents of this file are subject to the KnowledgeTree Public
+ * License Version 1.1.2 ("License"); You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.knowledgetree.com/KPL
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; using version 2 of the License.
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
+ * See the License for the specific language governing rights and
+ * limitations under the License.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+ * All copies of the Covered Code must include on each user interface screen:
+ *    (i) the "Powered by KnowledgeTree" logo and
+ *    (ii) the KnowledgeTree copyright notice
+ * in the same form as they appear in the distribution.  See the License for
+ * requirements.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * The Original Code is: KnowledgeTree Open Source
  *
- * -------------------------------------------------------------------------
- *
- * You can contact the copyright owner regarding licensing via the contact
- * details that can be found on the KnowledgeTree web site:
- *
- *         http://www.knowledgetree.com/
+ * The Initial Developer of the Original Code is The Jam Warehouse Software
+ * (Pty) Ltd, trading as KnowledgeTree.
+ * Portions created by The Jam Warehouse Software (Pty) Ltd are Copyright
+ * (C) 2007 The Jam Warehouse Software (Pty) Ltd;
+ * All Rights Reserved.
+ * Contributor( s): ______________________________________
  */
 class FolderUsageDashlet extends KTBaseDashlet
@@ -31,7 +35,7 @@ class FolderUsageDashlet extends KTBaseDashlet
 	function FolderUsageDashlet()
 	{
-		$this->sTitle = _kt('System Folder Usage');
+		$this->sTitle = _kt('System Folder Utilization');
 		$this->sClass = "ktInfo";
 	}
 <?php
+/**
+ * $Id
+ *
+ * The contents of this file are subject to the KnowledgeTree Public
+ * License Version 1.1.2 ("License"); You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.knowledgetree.com/KPL
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
+ * See the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * All copies of the Covered Code must include on each user interface screen:
+ *    (i) the "Powered by KnowledgeTree" logo and
+ *    (ii) the KnowledgeTree copyright notice
+ * in the same form as they appear in the distribution.  See the License for
+ * requirements.
+ *
+ * The Original Code is: KnowledgeTree Open Source
+ *
+ * The Initial Developer of the Original Code is The Jam Warehouse Software
+ * (Pty) Ltd, trading as KnowledgeTree.
+ * Portions created by The Jam Warehouse Software (Pty) Ltd are Copyright
+ * (C) 2007 The Jam Warehouse Software (Pty) Ltd;
+ * All Rights Reserved.
+ * Contributor( s): ______________________________________
+ */
+
 session_start();
 require_once("../../config/dmsDefaults.php");
@@ -88,4 +117,4 @@ class HouseKeeperDispatcher extends KTStandardDispatcher
 $oDispatcher = new HouseKeeperDispatcher();
 $oDispatcher->dispatch();
-?>
 \ No newline at end of file
+?>
 <?php
+/**
+ * $Id
+ *
+ * The contents of this file are subject to the KnowledgeTree Public
+ * License Version 1.1.2 ("License"); You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.knowledgetree.com/KPL
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
+ * See the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * All copies of the Covered Code must include on each user interface screen:
+ *    (i) the "Powered by KnowledgeTree" logo and
+ *    (ii) the KnowledgeTree copyright notice
+ * in the same form as they appear in the distribution.  See the License for
+ * requirements.
+ *
+ * The Original Code is: KnowledgeTree Open Source
+ *
+ * The Initial Developer of the Original Code is The Jam Warehouse Software
+ * (Pty) Ltd, trading as KnowledgeTree.
+ * Portions created by The Jam Warehouse Software (Pty) Ltd are Copyright
+ * (C) 2007 The Jam Warehouse Software (Pty) Ltd;
+ * All Rights Reserved.
+ * Contributor( s): ______________________________________
+ */
+
 class HouseKeeperPlugin extends KTPlugin
  {
 	var $autoRegister = true;
@@ -93,4 +122,4 @@ class HouseKeeperPlugin extends KTPlugin
 $oPluginRegistry =& KTPluginRegistry::getSingleton();
 $oPluginRegistry->registerPlugin('HouseKeeperPlugin', 'ktcore.housekeeper.plugin', __FILE__);
-?>
 \ No newline at end of file
+?>
@@ -124,10 +124,22 @@ abstract class DocumentExtractor
 		}
 		$classname=get_class($this);
+		$sql = "select id as extractor_id from mime_extractors WHERE name='$classname'";
+		$rs = DBUtil::getResultArray($sql);
+		if (count($rs) == 0)
+		{
+			$extractor_id = DBUtil::autoInsert('mime_extractors', array('name'=>$classname, 'active'=>1));
+		}
+		else
+		{
+			$extractor_id = $rs[0]['extractor_id'];
+		}
+
+
 		foreach($types as $type)
 		{
-			$sql = "update mime_types set extractor='$classname'  where mimetypes='$type' and extractor is null";
-			DBUtil::runQuery($sql);
+			$sql = "update mime_types set extractor_id=$extractor_id where mimetypes='$type' and extractor_id is null";
+			$rs = DBUtil::runQuery($sql);
 		}
 	}
@@ -510,15 +522,17 @@ abstract class CompositeExtractor extends DocumentExtractor
 	public function extractTextContent()
 	{
 		$intermediateFile = $this->targetfile . '.' . $this->targetExtension;
+		touch($intermediateFile);
 		$this->sourceExtractor->setSourceFile($this->sourcefile);
 		$this->sourceExtractor->setTargetFile($intermediateFile);
 		$this->sourceExtractor->setMimeType($this->mimetype);
 		$this->sourceExtractor->setExtension($this->extension);
-		if ($this->sourceExtractor->extractTextContent())
+		if (!$this->sourceExtractor->extractTextContent())
 		{
 			return false;
 		}
+		$intermediateFile = $this->sourceExtractor->getTargetFile();
 		$this->targetExtractor->setSourceFile($intermediateFile);
 		$this->targetExtractor->setTargetFile($this->targetfile);
 <?php
-require_once('OOPDFTextExtractor.inc.php');
+require_once('OOTextExtractor.inc.php');
+require_once('PDFExtractor.inc.php');
-class OOPresentationExtractor extends OOPDFTextExtractor
+class OOPresentationExtractor extends CompositeExtractor
 {
+	public function __construct()
+	{
+		$sourceExtractor = new OOPresentationToPDF();
+		$targetExtractor = new PDFExtractor();
+		parent::__construct($sourceExtractor, 'pdf', 'application/pdf', $targetExtractor, true);
+	}
+
 	public function getDisplayName()
 	{
 		return _kt('OpenOffice Presentation Extractor');
@@ -18,4 +26,18 @@ class OOPresentationExtractor extends OOPDFTextExtractor
 	}
 }
+class OOPresentationToPDF extends OOTextExtractor
+{
+	public function __construct()
+	{
+		parent::__construct('pdf');
+		$this->documentConverter = KT_DIR . '/bin/openoffice/pdfgen.py';
+		if (!is_file($this->documentConverter))
+		{
+			$this->documentConverter = false;
+		}
+	}
+}
+
+
 ?>
 \ No newline at end of file
 <?php
-require_once('OOPDFTextExtractor.inc.php');
+require_once('OOTextExtractor.inc.php');
-class OOSpreadsheetExtractor extends OOPDFTextExtractor
+class OOSpreadsheetExtractor extends OOTextExtractor
 {
 	public function getDisplayName()
 	{
@@ -2,22 +2,27 @@
 class OOTextExtractor extends ExternalDocumentExtractor
 {
-	private $converter;
-	private $javaPath;
-	private $ooHost;
-	private $ooPort;
-	private $targetMimeType;
+	protected $python;
+	protected $documentConverter;
+	protected $ooHost;
+	protected $ooPort;
+	protected $targetExtension;
-	public function __construct($targetMimeType='plain/text')
+	public function __construct($targetExtension='html')
 	{
 		parent::__construct();
+		$this->targetExtension = $targetExtension;
 		$config =& KTConfig::getSingleton();
-		$this->converter = KTUtil::findCommand('extractors/jodconverter', 'jodconverter');
-		$this->javaPath = KTUtil::findCommand('extractors/java', 'java');
-		$this->ooHost = $config->get('openoffice/host', 'localhost');
-		$this->ooPort = $config->get('openoffice/port', 8100);
-		$this->targetMimeType = $targetMimeType;
+		$this->python = KTUtil::findCommand('externalBinary/python');
+		$this->ooHost = $config->get('openoffice/host');
+		$this->ooPort = $config->get('openoffice/port');
+
+		$this->documentConverter = KT_DIR . '/bin/openoffice/DocumentConverter.py';
+		if (!is_file($this->documentConverter))
+		{
+			$this->documentConverter = false;
+		}
 	}
 	public function getDisplayName()
@@ -28,53 +33,78 @@ class OOTextExtractor extends ExternalDocumentExtractor
 	public function getSupportedMimeTypes()
 	{
 		return array(
-       		'text/rtf',
-       		'application/vnd.oasis.opendocument.text',
-       		'application/vnd.oasis.opendocument.text-template',
-       		'application/vnd.oasis.opendocument.text-web',
-       		'application/vnd.oasis.opendocument.text-master',
-       		'application/vnd.sun.xml.writer',
-       		'application/vnd.sun.xml.writer.template',
-       		'application/vnd.sun.xml.writer.global',
+
 		);
 	}
 	public function needsIntermediateSourceFile()
 	{
 		// we need the intermediate file because it
-		// has the correct extension. jodconverter uses the extension to determine mimetype
+		// has the correct extension. documentConverter uses the extension to determine mimetype
 		return true;
 	}
 	protected function getCommandLine()
 	{
-		$cmdline = "$this->javaPath -jar $this->converter $this->sourcefile $this->mimetype $this->targetfile $this->targetMimeType $this->ooHost $this->ooPort";
+		$sourcefile = escapeshellcmd($this->sourcefile);
+		unlink($this->targetfile);
+		$this->targetfile .= '.' . $this->targetExtension;
+		$targetfile = escapeshellcmd($this->targetfile);
+
+		$escape = OS_WINDOWS?'"':'\'';
+
+		$cmdline = "{$this->python} {$escape}{$this->documentConverter}{$escape} {$escape}{$sourcefile}{$escape} {$escape}{$targetfile}{$escape} {$this->ooHost} {$this->ooPort}";
 		return $cmdline;
 	}
-	public function diagnose()
+	protected function filter($text)
 	{
-		if (false === $this->converter)
+		 $text = preg_replace ("@(</?[^>]*>)+@", '', $text);
+
+		 do
+		 {
+			 $old = $text;
+
+			 $text= preg_replace("@([\r\n])[\s]+@",'\1', $text);
+
+			 $text = preg_replace('@\ \ @',' ', $text);
+			 $text = preg_replace("@\n\n@","\n", $text);
+		 }
+		 while ($old != $text);
+
+		 return $text;
+	}
+
+	public function extractTextContent()
+	{
+		if (false === parent::extractTextContent())
 		{
-			return _kt('Cannot locate jodconverter');
+			return false;
 		}
-		if (false === $this->javaPath)
+		if ($this->targetExtension != 'html')
 		{
-			return _kt('Cannot locate java');
+			return true;
 		}
+		$content = file_get_contents($this->targetfile);
+		return file_put_contents($this->targetfile, $this->filter($content));
+	}
-		$connection = @fsockopen($this->ooHost, $this->ooPort,$errno, $errstr,5 );
-		if (false === $connection)
+	public function diagnose()
+	{
+		if (false === $this->python)
 		{
-			return _kt('Cannot connect to openoffice host');
+			return _kt('Cannot locate python');
 		}
-		fclose($connection);
+		if (false === $this->documentConverter)
+		{
+			return _kt('Cannot locate DocumentConverter.py');
+		}
-		return null;
+		return SearchHelper::checkOpenOfficeAvailablity();
 	}
 }
@@ -4,7 +4,7 @@ class PDFExtractor extends ApplicationExtractor
 {
 	public function __construct()
 	{
-		parent::__construct('externalBinary','pdftotext','pdftotext',_kt('PDF Text Extractor'),'-nopgbrk -enc UTF-8 {source} {target}');
+		parent::__construct('externalBinary','pdftotext','pdftotext',_kt('PDF Text Extractor'),'-nopgbrk -enc UTF-8 \'{source}\' \'{target}\'');
 	}
 	public function getSupportedMimeTypes()
 <?php
-class MailMimeExtractor extends TextExtractor
+require_once('OOTextExtractor.inc.php');
+
+class RTFExtractor extends OOTextExtractor
 {
 	public function getDisplayName()
 	{
-		return _kt('Mail Mime Extractor');
+		return _kt('RTF Extractor');
 	}
 	public function getSupportedMimeTypes()
 	{
-		return array('text/msg');
+		return array(
+       		'text/rtf'
+		);
 	}
-
 }
+
 ?>
 \ No newline at end of file
@@ -9,7 +9,7 @@ class XMLExtractor extends TextExtractor
 	public function getSupportedMimeTypes()
 	{
-		return array('text/xml','application/xml','text/html');
+		return array('text/xml','application/xml','text/html','text/enriched');
 	}
 	protected function filter($text)
@@ -330,7 +330,11 @@ abstract class Indexer
 	public function clearExtractors()
 	{
 		global $default;
-		$sql = "update mime_types set extractor=null";
+
+		$sql = "update mime_types set extractor_id=null";
+		DBUtil::runQuery($sql);
+
+		$sql = "delete from mime_extractors";
 		DBUtil::runQuery($sql);
 		$default->log->debug('clearExtractors');
@@ -616,6 +620,43 @@ abstract class Indexer
     }
     /**
+     * This does the initial mime type association between mime types and text extractors
+     *
+     */
+    public function checkForRegisteredTypes()
+    {
+    	global $default;
+
+    	// we are only doing this once!
+    	$initRegistered = KTUtil::getSystemSetting('mimeTypesRegistered', false);
+    	if ($initRegistered)
+    	{
+    		return;
+    	}
+    	$default->log->info('checkForRegisteredTypes: start');
+
+    	$this->registerTypes(true);
+
+
+    	$disable = array(
+    		OS_WINDOWS=>array('PSExtractor'),
+    		OS_UNIX => array()
+
+    	);
+
+		foreach($disable[OS_WINDOWS] as $extractor)
+		{
+    		$sql = "UPDATE mime_extractors SET active=0 WHERE name='$extractor'";
+    		DBUtil::runQuery($sql);
+    		$default->log->info("checkForRegisteredTypes: disabled '$extractor'");
+    	}
+
+    	$default->log->info('checkForRegisteredTypes: done');
+    	KTUtil::setSystemSetting('mimeTypesRegistered', true);
+    }
+
+
+    /**
      * The main function that may be called repeatedly to index documents.
      *
      * @param int $max Default 20
@@ -624,6 +665,8 @@ abstract class Indexer
     {
     	global $default;
+    	$this->checkForRegisteredTypes();
+
     	$default->log->info('indexDocuments: start');
     	if (!$this->doesDiagnosticsPass())
     	{
@@ -798,6 +841,9 @@ abstract class Indexer
         		if ($extractor->extractTextContent())
         		{
+        			// the extractor may need to create another target file
+        			$targetFile = $extractor->getTargetFile();
+
         			$extractor->setExtractionStatus(true);
         			$this->executeHook($extractor, 'pre_index');
 					$this->executeHook($extractor, 'pre_index', $mimeType);