Peter M. Groen / knowledgetree

Commit 4f809981a4726dd0053e7670ca28066005f21e94

Authored by Megan Watson 2008-10-09 07:39:53 +0000

KTS-3806

"The number of documents in the indexing queue is incorrect on the Document Indexer Statistics dashlet"
Fixed. Adjusted sql to ignore problem documents.

Committed by: Megan Watson
Reviewed by: Conrad Vermeulen

git-svn-id: https://kt-dms.svn.sourceforge.net/svnroot/kt-dms/trunk@9511 c91229c3-7414-0410-bfa2-8a42b809f60b

Inline Side-by-side

Showing 1 changed file with 1862 additions and 1862 deletions

search2/indexing/indexerCore.inc.php

View file @4f80998

1	-<?php
2	-
3	-/**
4	- * $Id:$
5	- *
6	- * KnowledgeTree Community Edition
7	- * Document Management Made Simple
8	- * Copyright (C) 2008 KnowledgeTree Inc.
9	- * Portions copyright The Jam Warehouse Software (Pty) Limited
10	- *
11	- * This program is free software; you can redistribute it and/or modify it under
12	- * the terms of the GNU General Public License version 3 as published by the
13	- * Free Software Foundation.
14	- *
15	- * This program is distributed in the hope that it will be useful, but WITHOUT
16	- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17	- * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
18	- * details.
19	- *
20	- * You should have received a copy of the GNU General Public License
21	- * along with this program. If not, see <http://www.gnu.org/licenses/>.
22	- *
23	- * You can contact KnowledgeTree Inc., PO Box 7775 #87847, San Francisco,
24	- * California 94120-7775, or email info@knowledgetree.com.
25	- *
26	- * The interactive user interfaces in modified source and object code versions
27	- * of this program must display Appropriate Legal Notices, as required under
28	- * Section 5 of the GNU General Public License version 3.
29	- *
30	- * In accordance with Section 7(b) of the GNU General Public License version 3,
31	- * these Appropriate Legal Notices must retain the display of the "Powered by
32	- * KnowledgeTree" logo and retain the original copyright notice. If the display of the
33	- * logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices
34	- * must display the words "Powered by KnowledgeTree" and retain the original
35	- * copyright notice.
36	- * Contributor( s): ______________________________________
37	- *
38	- */
39	-
40	-define('SEARCH2_INDEXER_DIR',realpath(dirname(__FILE__)) . '/');
41	-require_once('indexing/extractorCore.inc.php');
42	-require_once(KT_DIR . '/plugins/ktcore/scheduler/schedulerUtil.php');
43	-require_once(KT_DIR . '/ktapi/ktapi.inc.php');
44	-
45	-class IndexerInconsistencyException extends Exception {};
46	-
47	-// TODO: Query Result Items code should be moved into the Search section. It has less to do with indexing...
48	-
49	-class QueryResultItem
50	-{
51	- protected $id;
52	- protected $title;
53	- protected $rank;
54	- protected $text;
55	- protected $fullpath;
56	-
57	- public function __construct($id, $title, $rank, $text, $fullpath)
58	- {
59	- $this->id = $id;
60	- $this->title = $title;
61	- $this->rank = $rank;
62	- $this->text = $text;
63	- $this->fullpath = $fullpath;
64	- }
65	-
66	- public function getId() { return $this->id; }
67	- public function getRealId() { return $this->id; }
68	-
69	- public function getIsProxy() { return $this instanceof ProxyResultItem; }
70	- public function getIsFolder() { return substr(get_class($this), 0, 6) == 'Folder' ; }
71	- public function getIsDocument() { return substr(get_class($this), 0, 8) == 'Document' ; }
72	-
73	- public function setRank($value)
74	- {
75	- $this->rank = number_format($value,2,'.',',');
76	- }
77	-
78	- public function getIsLive()
79	- {
80	- return true;
81	- }
82	-
83	- public function setTitle($value)
84	- {
85	- $this->title = $value;
86	- }
87	-
88	- public function setText($value)
89	- {
90	- $this->text = $value;
91	- }
92	-
93	- public function getRelevance() { return (float) $this->rank; }
94	- public function getRank() { return $this->getRelevance(); }
95	- public function getText() { return (string) $this->text; }
96	- public function getTitle() { return (string) $this->title; }
97	- public function getFullPath() { return (string) $this->fullpath; }
98	-
99	- protected function __get($property)
100	- {
101	- if (empty($property))
102	- {
103	- return '';
104	- }
105	-
106	- $method = 'get' . $property;
107	- if (method_exists($this, $method))
108	- {
109	- return $this->$method();
110	- }
111	- return $this->getUnknown();
112	- }
113	-
114	- protected function getUnknown()
115	- {
116	- return _kt('n/a');
117	- }
118	-
119	- protected function __set($property, $value)
120	- {
121	- if (empty($property))
122	- {
123	- return '';
124	- }
125	-
126	- $method = 'set' . $property;
127	- if (method_exists($this, $method))
128	- {
129	- return $this->$method($value);
130	- }
131	- throw new Exception("Unknown property '$property' to set on QueryResultItem");
132	- }
133	-}
134	-
135	-class ProxyResultItem extends QueryResultItem
136	-{
137	- protected $proxy;
138	- protected $proxyId;
139	-
140	- public function __construct($proxyId, $proxy)
141	- {
142	- parent::__construct($proxyId, $proxy->getTitle, $proxy->getRank(), $proxy->getText(), $proxy->getFullPath());
143	- $this->proxyId = $proxyId;
144	- $this->proxy = $proxy;
145	- }
146	-
147	- public function getId() { return $this->proxyId; }
148	- public function getTitle() { return $this->proxy->getTitle(); }
149	- public function getRealId() { return $this->proxy->getId(); }
150	-
151	- protected function __get($property)
152	- {
153	- $method = 'get' . $property;
154	-
155	- if (method_exists($this, $method))
156	- {
157	- return $this->$method();
158	- }
159	- else
160	- {
161	- return $this->proxy->$method();
162	- }
163	- }
164	-
165	- protected function __set($property, $value)
166	- {
167	- $method = 'set' . $property;
168	- if (method_exists($this, $method))
169	- {
170	- return $this->$method($value);
171	- }
172	- else
173	- {
174	- return $this->proxy->$method($value);
175	- }
176	- }
177	-}
178	-
179	-class DocumentResultItem extends QueryResultItem
180	-{
181	- protected $filesize;
182	- protected $live;
183	- protected $version;
184	- protected $mimeType;
185	- protected $filename;
186	- protected $thumbnail; // TODO: if not null, gui can display a thumbnail
187	- protected $viewer; // TODO: if not null, a viewer can be used to view the document
188	- protected $document;
189	- protected $checkedOutUser;
190	- protected $dateCheckedout;
191	- protected $workflowState;
192	- protected $workflow;
193	- protected $modifiedBy;
194	- protected $dateModified;
195	- protected $createdBy;
196	- protected $dateCreated;
197	- protected $owner;
198	- protected $immutable;
199	- protected $deleted;
200	- protected $status;
201	- protected $folderId;
202	- protected $storagePath;
203	- protected $documentType;
204	- protected $mimeIconPath;
205	- protected $mimeDisplay;
206	- protected $oemDocumentNo;
207	-
208	- public function __construct($document_id, $rank=null, $title=null, $text=null, $fullpath = null)
209	- {
210	- parent::__construct($document_id, $title, $rank, $text, $fullpath);
211	- $this->live = true;
212	- $this->loadDocumentInfo();
213	- }
214	-
215	- // TODO: this is bad. must refactor to do the query on the group of documents.
216	- public function loadDocumentInfo()
217	- {
218	- global $default;
219	- $sql = "SELECT
220	- d.folder_id, f.full_path, f.name, dcv.size as filesize, dcv.major_version,
221	- dcv.minor_version, dcv.filename, cou.name as checkoutuser, w.human_name as workflow, ws.human_name as workflowstate,
222	- mt.mimetypes as mimetype, md.mime_doc as mimedoc, d.checkedout, mbu.name as modifiedbyuser, d.modified,
223	- cbu.name as createdbyuser, ou.name as owneruser, d.immutable, d.status_id, d.created,dcv.storage_path, dtl.name as document_type,
224	- mt.icon_path as mime_icon_path, mt.friendly_name as mime_display, d.oem_no, dmv.name as title
225	- FROM
226	- documents d
227	- INNER JOIN document_metadata_version dmv ON d.metadata_version_id = dmv.id
228	- INNER JOIN document_content_version dcv ON dmv.content_version_id = dcv.id
229	- INNER JOIN mime_types mt ON dcv.mime_id=mt.id
230	- LEFT JOIN document_types_lookup dtl ON dtl.id=dmv.document_type_id
231	- LEFT JOIN folders f ON f.id=d.folder_id
232	- LEFT JOIN users cou ON d.checked_out_user_id=cou.id
233	- LEFT JOIN workflows w ON dmv.workflow_id=w.id
234	- LEFT JOIN workflow_states ws ON dmv.workflow_state_id = ws.id
235	- LEFT JOIN mime_documents md ON mt.mime_document_id = md.id
236	- LEFT JOIN users mbu ON d.modified_user_id=mbu.id
237	- LEFT JOIN users cbu ON d.creator_id=cbu.id
238	- LEFT JOIN users ou ON d.owner_id=ou.id
239	- WHERE
240	- d.id=$this->id";
241	-
242	- $result = DBUtil::getOneResult($sql);
243	-
244	- if (PEAR::isError($result) \|\| empty($result))
245	- {
246	- $this->live = false;
247	- if (PEAR::isError($result))
248	- {
249	- throw new Exception('Database exception! There appears to be an error in the system: ' .$result->getMessage());
250	- }
251	-
252	- $default->log->error('QueryResultItem: $result is null');
253	- $msg = 'The database did not have a record matching the result from the document indexer. This may occur if there is an inconsistency between the document indexer and the repository. The indexer needs to be repaired.';
254	- $default->log->error('QueryResultItem: ' . $msg);
255	- // TODO: repair process where we scan documents in index, and delete those for which there is nothing in the repository
256	- throw new IndexerInconsistencyException(_kt($msg));
257	- }
258	-
259	- // document_id, relevance, text, title
260	-
261	- $this->documentType = $result['document_type'];
262	- $this->filename=$result['filename'];
263	- $this->filesize = KTUtil::filesizeToString($result['filesize']);
264	- $this->folderId = $result['folder_id'];
265	- $this->title = $result['title'];
266	-
267	- $this->createdBy = $result['createdbyuser'];
268	- $this->dateCreated = $result['created'];
269	-
270	- $this->modifiedBy = $result['modifiedbyuser'];
271	- $this->dateModified = $result['modified'];
272	-
273	- $this->checkedOutUser = $result['checkoutuser'];
274	- $this->dateCheckedout = $result['checkedout'];
275	-
276	- $this->owner = $result['owneruser'];
277	-
278	- $this->version = $result['major_version'] . '.' . $result['minor_version'];
279	-
280	- $this->immutable = ($result['immutable'] + 0)?_kt('Immutable'):'';
281	-
282	- $this->workflow = $result['workflow'];
283	- $this->workflowState = $result['workflowstate'];
284	-
285	- $this->oemDocumentNo = $result['oem_no'];
286	- if (empty($this->oemDocumentNo)) $this->oemDocumentNo = 'n/a';
287	-
288	- if (is_null($result['name']))
289	- {
290	- $this->fullpath = '(orphaned)';
291	- }
292	- else
293	- {
294	- $this->fullpath = $result['full_path'];
295	- }
296	-
297	- $this->mimeType = $result['mimetype'];
298	- $this->mimeIconPath = $result['mime_icon_path'];
299	- if (empty($this->mimeIconPath))
300	- {
301	- $this->mimeIconPath = 'unspecified_type';
302	- }
303	- $this->mimeDisplay = $result['mime_display'];
304	-
305	- $this->storagePath = $result['storage_path'];
306	- $this->status = Document::getStatusString($result['status_id']);
307	- }
308	-
309	- public function getDocumentID() { return $this->getId(); }
310	- public function getIsLive() { return (bool) $this->live; }
311	- public function getFilesize() { return $this->filesize; }
312	- public function getVersion() { return (string) $this->version; }
313	- public function getFilename() { return (string)$this->filename; }
314	- public function getFolderId() { return (int)$this->folderId; }
315	- public function getOemDocumentNo() { return (string) $this->oemDocumentNo; }
316	- public function getDocument() { return Document::get($this->id); }
317	- public function getIsAvailable() { return $this->Document->isLive(); }
318	- public function getCheckedOutUser() { return (string) $this->checkedOutUser; }
319	- public function getCheckedOutByr() { return $this->getCheckedOutUser(); }
320	- public function getWorkflowOnly() { return (string)$this->workflow; }
321	- public function getWorkflow() { return $this->getWorkflow(); }
322	- public function getWorkflowStateOnly() { return (string)$this->workflowState; }
323	- public function getWorkflowState() { return $this->getWorkflowStateOnly(); }
324	- public function getWorkflowAndState() {
325	- if (is_null($this->workflow))
326	- {
327	- return '';
328	- }
329	- return "$this->workflow - $this->workflowState";
330	- }
331	- public function getMimeType() { return (string) $this->mimeType; }
332	- public function getMimeIconPath() { return (string) $this->mimeIconPath; }
333	- public function getMimeDisplay() { return (string) $this->mimeDisplay; }
334	- public function getDateCheckedOut() { return (string) $this->dateCheckedout; }
335	- public function getModifiedBy() { return (string) $this->modifiedBy; }
336	- public function getDateModified() { return (string) $this->dateModified; }
337	- public function getCreatedBy() { return (string) $this->createdBy; }
338	- public function getDateCreated() { return (string) $this->dateCreated; }
339	- public function getOwner() { return (string) $this->owner; }
340	- public function getOwnedBy() { return $this->getOwner(); }
341	- public function getIsImmutable() { return (bool) $this->immutable; }
342	- public function getImmutable() { return $this->getIsImmutable(); }
343	- public function getStatus() { return $this->status; }
344	- public function getStoragePath() { return $this->storagePath; }
345	- public function getDocumentType() { return $this->documentType; }
346	- public function getPermissions() { return KTAPI_Document::get_permission_string($this->Document); }
347	- public function getCanBeReadByUser() {
348	- if (!$this->live)
349	- return false;
350	- if (Permission::userHasDocumentReadPermission($this->Document))
351	- return true;
352	- if (Permission::adminIsInAdminMode())
353	- return true;
354	- return false;
355	- }
356	-}
357	-
358	-class FolderResultItem extends QueryResultItem
359	-{
360	- protected $folder;
361	- protected $createdBy;
362	- protected $parentId;
363	-
364	- public function __construct($folder_id, $rank=null, $title=null, $text=null, $fullpath = null)
365	- {
366	- parent::__construct($folder_id, $title, $rank, $text, $fullpath);
367	- $this->loadFolderInfo();
368	- }
369	-
370	- public function getFolderID() { return $this->getId(); }
371	- public function getParentID() { return $this->parentId; }
372	- public function getCreatedBy() { return $this->createdBy; }
373	- public function getMimeIconPath() { return 'folder'; }
374	- public function getFolder() { return Folder::get($this->getFolderID()); }
375	- public function getPermissions() { return KTAPI_Folder::get_permission_string($this->Folder); }
376	-
377	- public function loadFolderInfo()
378	- {
379	- global $default;
380	- $folder = $this->getFolder();
381	- if (PEAR::isError($folder))
382	- {
383	- throw new Exception('Database exception! There appears to be an error in the system: ' .$result->getMessage());
384	- }
385	- $this->title = $folder->getName();
386	- $this->fullpath = '/' . $folder->getFullPath();
387	- $this->parentId = $folder->getParentId();
388	-
389	- $user = User::get($folder->getCreatorID());
390	- $this->createdBy = (PEAR::isError($user))?_kt('Unknown'):$user->getName();
391	- }
392	-
393	-}
394	-
395	-class DocumentShortcutResultItem extends ProxyResultItem
396	-{
397	- public function getDocumentID() { return $this->getId(); }
398	- public function getMimeIconPath() { return $this->proxy->getMimeIconPath() . '_shortcut'; }
399	-
400	-}
401	-
402	-class FolderShortcutResultItem extends ProxyResultItem
403	-{
404	- public function getFolderID() { return $this->getId(); }
405	- public function getMimeIconPath() { return 'folder_shortcut'; }
406	-
407	-}
408	-
409	-function MatchResultCompare($a, $b)
410	-{
411	- if ($a->Rank == $b->Rank) {
412	- return 0;
413	- }
414	- return ($a->Rank < $b->Rank) ? -1 : 1;
415	-}
416	-
417	-abstract class Indexer
418	-{
419	- /**
420	- * Cache of extractors
421	- *
422	- * @var array
423	- */
424	- private $extractorCache;
425	-
426	- /**
427	- * Indicates if the indexer will do logging.
428	- *
429	- * @var boolean
430	- */
431	- private $debug;
432	- /**
433	- * Cache on mime related hooks
434	- *
435	- * @var unknown_type
436	- */
437	- private $mimeHookCache;
438	- /**
439	- * Cache on general hooks.
440	- *
441	- * @var array
442	- */
443	- private $generalHookCache;
444	-
445	- /**
446	- * This is a path to the extractors.
447	- *
448	- * @var string
449	- */
450	- private $extractorPath;
451	- /**
452	- * This is a path to the hooks.
453	- *
454	- * @var string
455	- */
456	- private $hookPath;
457	-
458	- private $enabledExtractors;
459	-
460	- /**
461	- * Initialise the indexer
462	- *
463	- */
464	- protected function __construct()
465	- {
466	- $config = KTConfig::getSingleton();
467	-
468	- $this->extractorCache = array();
469	- $this->debug = $config->get('indexer/debug', true);
470	- $this->hookCache = array();
471	- $this->generalHookCache = array();
472	- $this->extractorPath = $config->get('indexer/extractorPath', 'extractors');
473	- $this->hookPath = $config->get('indexer/extractorHookPath','extractorHooks');
474	-
475	- $this->loadExtractorStatus();
476	- }
477	-
478	- /**
479	- * Get the list if enabled extractors
480	- *
481	- */
482	- private function loadExtractorStatus()
483	- {
484	- $sql = "SELECT id, name FROM mime_extractors WHERE active=1";
485	- $rs = DBUtil::getResultArray($sql);
486	- $this->enabledExtractors = array();
487	- foreach($rs as $item)
488	- {
489	- $this->enabledExtractors[] = $item['name'];
490	- }
491	- }
492	-
493	- private function isExtractorEnabled($extractor)
494	- {
495	- return in_array($extractor, $this->enabledExtractors);
496	- }
497	-
498	- /**
499	- * Returns a reference to the main class
500	- *
501	- * @return Indexer
502	- */
503	- public static function get()
504	- {
505	- static $singleton = null;
506	-
507	- if (is_null($singleton))
508	- {
509	- $config = KTConfig::getSingleton();
510	- $classname = $config->get('indexer/coreClass');
511	-
512	- require_once('indexing/indexers/' . $classname . '.inc.php');
513	-
514	- if (!class_exists($classname))
515	- {
516	- throw new Exception("Class '$classname' does not exist.");
517	- }
518	-
519	- $singleton = new $classname;
520	- }
521	-
522	- return $singleton;
523	- }
524	-
525	- public abstract function deleteDocument($docid);
526	-
527	- /**
528	- * Remove the association of all extractors to mime types on the database.
529	- *
530	- */
531	- public function clearExtractors()
532	- {
533	- global $default;
534	-
535	- $sql = "update mime_types set extractor_id=null";
536	- DBUtil::runQuery($sql);
537	-
538	- $sql = "delete from mime_extractors";
539	- DBUtil::runQuery($sql);
540	-
541	- if ($this->debug) $default->log->debug('clearExtractors');
542	- }
543	-
544	- /**
545	- * lookup the name of the extractor class based on the mime type.
546	- *
547	- * @param string $type
548	- * @return string
549	- */
550	- public static function resolveExtractor($type)
551	- {
552	- global $default;
553	- $sql = "select extractor from mime_types where filetypes='$type'";
554	- $class = DBUtil::getOneResultKey($sql,'extractor');
555	- if (PEAR::isError($class))
556	- {
557	- $default->log->error("resolveExtractor: cannot resolve $type");
558	- return $class;
559	- }
560	- if ($this->debug) $default->log->debug(sprintf(_kt("resolveExtractor: Resolved '%s' from mime type '%s'."), $class, $type));
561	- return $class;
562	- }
563	-
564	- /**
565	- * Return all the discussion text.
566	- *
567	- * @param int $docid
568	- * @return string
569	- */
570	- public static function getDiscussionText($docid)
571	- {
572	- $sql = "SELECT
573	- dc.subject, dc.body
574	- FROM
575	- discussion_threads dt
576	- INNER JOIN discussion_comments dc ON dc.thread_id=dt.id AND dc.id BETWEEN dt.first_comment_id AND dt.last_comment_id
577	- WHERE
578	- dt.document_id=$docid";
579	- $result = DBUtil::getResultArray($sql);
580	- $text = '';
581	-
582	- foreach($result as $record)
583	- {
584	- $text .= $record['subject'] . "\n" . $record['body'] . "\n";
585	- }
586	-
587	- return $text;
588	- }
589	-
590	- /**
591	- * Schedule the indexing of a document.
592	- *
593	- * @param string $document
594	- * @param string $what
595	- */
596	- public static function index($document, $what='A')
597	- {
598	- global $default;
599	-
600	- if (is_numeric($document))
601	- {
602	- $document = Document::get($document+0);
603	- }
604	-
605	- if (PEAR::isError($document))
606	- {
607	- $default->log->error("index: Could not index document: " .$document->getMessage());
608	- return;
609	- }
610	-
611	- $document_id = $document->getId();
612	- $userid=$_SESSION['userID'];
613	- if (empty($userid)) $userid=1;
614	-
615	- // we dequeue the document so that there are no issues when enqueuing
616	- Indexer::unqueueDocument($document_id);
617	-
618	- // enqueue item
619	- $sql = "INSERT INTO index_files(document_id, user_id, what) VALUES($document_id, $userid, '$what')";
620	- DBUtil::runQuery($sql);
621	-
622	- $default->log->debug("index: Queuing indexing of $document_id");
623	-
624	- }
625	-
626	- private static function incrementCount()
627	- {
628	- // Get count from system settings
629	- $count = Indexer::getIndexedDocumentCount();
630	- $count = (int)$count + 1;
631	- Indexer::updateIndexedDocumentCount($count);
632	- }
633	-
634	- public static function getIndexedDocumentCount()
635	- {
636	- $count = KTUtil::getSystemSetting('indexedDocumentCount', 0);
637	- return (int) $count;
638	- }
639	-
640	- public static function updateIndexedDocumentCount($cnt = 0)
641	- {
642	- KTUtil::setSystemSetting('indexedDocumentCount', $cnt);
643	- }
644	-
645	- public static function reindexQueue()
646	- {
647	- $sql = "UPDATE index_files SET processdate = null";
648	- DBUtil::runQuery($sql);
649	- }
650	-
651	- public static function reindexDocument($documentId)
652	- {
653	- $sql = "UPDATE index_files SET processdate=null, status_msg=null WHERE document_id=$documentId";
654	- DBUtil::runQuery($sql);
655	- }
656	-
657	-
658	-
659	- public static function indexAll()
660	- {
661	- $userid=$_SESSION['userID'];
662	- if (empty($userid)) $userid=1;
663	-
664	- $sql = "DELETE FROM index_files";
665	- DBUtil::runQuery($sql);
666	-
667	- $sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'A' FROM documents WHERE status_id=1 and id not in (select document_id from index_files)";
668	- DBUtil::runQuery($sql);
669	- }
670	-
671	- public static function indexFolder($folder)
672	- {
673	- $userid=$_SESSION['userID'];
674	- if (empty($userid)) $userid=1;
675	-
676	- if (!$folder instanceof Folder && !$folder instanceof FolderProxy)
677	- {
678	- throw new Exception('Folder expected');
679	- }
680	-
681	- $full_path = $folder->getFullPath();
682	-
683	- $sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'A' FROM documents WHERE full_path like '{$full_path}/%' AND status_id=1 and id not in (select document_id from index_files)";
684	- DBUtil::runQuery($sql);
685	- }
686	-
687	- /**
688	- * Clearout the scheduling of documents that no longer exist.
689	- *
690	- */
691	- public static function clearoutDeleted()
692	- {
693	- global $default;
694	-
695	- $sql = 'DELETE FROM
696	- index_files
697	- WHERE
698	- document_id in (SELECT d.id FROM documents AS d WHERE d.status_id=3) OR
699	- NOT EXISTS(SELECT index_files.document_id FROM documents WHERE index_files.document_id=documents.id)';
700	- DBUtil::runQuery($sql);
701	-
702	- $default->log->debug("Indexer::clearoutDeleted: removed documents from indexing queue that have been deleted");
703	- }
704	-
705	-
706	- /**
707	- * Check if a document is scheduled to be indexed
708	- *
709	- * @param mixed $document This may be a document or document id
710	- * @return boolean
711	- */
712	- public static function isDocumentScheduled($document)
713	- {
714	- if (is_numeric($document))
715	- {
716	- $docid = $document;
717	- }
718	- else if ($document instanceof Document)
719	- {
720	- $docid = $document->getId();
721	- }
722	- else
723	- {
724	- return false;
725	- }
726	- $sql = "SELECT 1 FROM index_files WHERE document_id=$docid";
727	- $result = DBUtil::getResultArray($sql);
728	- return count($result) > 0;
729	- }
730	-
731	- /**
732	- * Filters text removing redundant characters such as continuous newlines and spaces.
733	- *
734	- * @param string $filename
735	- */
736	- private function filterText($filename)
737	- {
738	- $content = file_get_contents($filename);
739	-
740	- $src = array("([\r\n])","([\n][\n])","([\n])","([\t])",'([ ][ ])');
741	- $tgt = array("\n","\n",' ',' ',' ');
742	-
743	- // shrink what is being stored.
744	- do
745	- {
746	- $orig = $content;
747	- $content = preg_replace($src, $tgt, $content);
748	- } while ($content != $orig);
749	-
750	- return file_put_contents($filename, $content) !== false;
751	- }
752	-
753	- /**
754	- * Load hooks for text extraction process.
755	- *
756	- */
757	- private function loadExtractorHooks()
758	- {
759	- $this->generalHookCache = array();
760	- $this->mimeHookCache = array();
761	-
762	-
763	- $dir = opendir(SearchHelper::correctPath($this->hookPath));
764	- while (($file = readdir($dir)) !== false)
765	- {
766	- if (substr($file,-12) == 'Hook.inc.php')
767	- {
768	- require_once($this->hookPath . '/' . $file);
769	- $class = substr($file, 0, -8);
770	-
771	- if (!class_exists($class))
772	- {
773	- continue;
774	- }
775	-
776	- $hook = new $class;
777	- if (!($class instanceof ExtractorHook))
778	- {
779	- continue;
780	- }
781	-
782	- $mimeTypes = $hook->registerMimeTypes();
783	- if (is_null($mimeTypes))
784	- {
785	- $this->generalHookCache[] = & $hook;
786	- }
787	- else
788	- {
789	- foreach($mimeTypes as $type)
790	- {
791	- $this->mimeHookCache[$type][] = & $hook;
792	- }
793	- }
794	-
795	- }
796	- }
797	- closedir($dir);
798	- }
799	-
800	- /**
801	- * This is a refactored function to execute the hooks.
802	- *
803	- * @param DocumentExtractor $extractor
804	- * @param string $phase
805	- * @param string $mimeType Optional. If set, indicates which hooks must be used, else assume general.
806	- */
807	- private function executeHook($extractor, $phase, $mimeType = null)
808	- {
809	- $hooks = array();
810	- if (is_null($mimeType))
811	- {
812	- $hooks = $this->generalHookCache;
813	- }
814	- else
815	- {
816	- if (array_key_exists($mimeType, $this->mimeHookCache))
817	- {
818	- $hooks = $this->mimeHookCache[$mimeType];
819	- }
820	- }
821	- if (empty($hooks))
822	- {
823	- return;
824	- }
825	-
826	- foreach($hooks as $hook)
827	- {
828	- $hook->$phase($extractor);
829	- }
830	- }
831	-
832	- private function doesDiagnosticsPass($simple=false)
833	- {
834	- global $default;
835	-
836	- $config =& KTConfig::getSingleton();
837	- // create a index log lock file in case there are errors, and we don't need to log them forever!
838	- // this function will create the lockfile if an error is detected. It will be removed as soon
839	- // as the problems with the indexer are removed.
840	- $lockFile = $config->get('cache/cacheDirectory') . '/index.log.lock';
841	-
842	- $diagnosis = $this->diagnose();
843	- if (!is_null($diagnosis))
844	- {
845	- if (!is_file($lockFile))
846	- {
847	- $default->log->error(_kt('Indexer problem: ') . $diagnosis);
848	- }
849	- touch($lockFile);
850	- return false;
851	- }
852	-
853	- if ($simple)
854	- {
855	- return true;
856	- }
857	-
858	- $diagnosis = $this->diagnoseExtractors();
859	- if (!empty($diagnosis))
860	- {
861	- if (!is_file($lockFile))
862	- {
863	- foreach($diagnosis as $diag)
864	- {
865	- $default->log->error(sprintf(_kt('%s problem: %s'), $diag['name'],$diag['diagnosis']));
866	- }
867	- }
868	- touch($lockFile);
869	- return false;
870	- }
871	-
872	- if (is_file($lockFile))
873	- {
874	- $default->log->info(_kt('Issues with the indexer have been resolved!'));
875	- unlink($lockFile);
876	- }
877	-
878	- return true;
879	- }
880	-
881	- /**
882	- * This does the initial mime type association between mime types and text extractors
883	- *
884	- */
885	- public function checkForRegisteredTypes()
886	- {
887	- global $default;
888	-
889	- // we are only doing this once!
890	- $initRegistered = KTUtil::getSystemSetting('mimeTypesRegistered', false);
891	- if ($initRegistered)
892	- {
893	- return;
894	- }
895	- if ($this->debug) $default->log->debug('checkForRegisteredTypes: start');
896	-
897	- $date = date('Y-m-d H:i');
898	- $sql = "UPDATE scheduler_tasks SET run_time='$date'";
899	- DBUtil::runQuery($sql);
900	-
901	- $this->registerTypes(true);
902	-
903	- $disable = array(
904	- 'windows'=>array('PSExtractor'),
905	- 'unix' => array()
906	- );
907	-
908	- $disableForOS = OS_WINDOWS?$disable['windows']:$disable['unix'];
909	-
910	- if (!empty($disableForOS))
911	- {
912	- $disableForOS = '\'' . implode("','", $disableForOS) .'\'';
913	-
914	- $sql = "UPDATE mime_extractors SET active=0 WHERE name in ($disableForOS)";
915	- DBUtil::runQuery($sql);
916	- $default->log->info("checkForRegisteredTypes: disabled '$extractor'");
917	- }
918	- $this->loadExtractorStatus();
919	-
920	- if ($this->debug) $default->log->debug('checkForRegisteredTypes: done');
921	- KTUtil::setSystemSetting('mimeTypesRegistered', true);
922	- }
923	-
924	- private function updatePendingDocumentStatus($documentId, $message, $level)
925	- {
926	- $this->indexingHistory .= "\n" . $level . ': ' . $message;
927	- $message = sanitizeForSQL($this->indexingHistory);
928	- $sql = "UPDATE index_files SET status_msg='$message' WHERE document_id=$documentId";
929	- DBUtil::runQuery($sql);
930	- }
931	-
932	- private $restartCurrentBatch = false;
933	-
934	- public function restartBatch()
935	- {
936	- $this->restartCurrentBatch = true;
937	- }
938	-
939	- /**
940	- *
941	- * @param int $documentId
942	- * @param string $message
943	- * @param string $level This may be info, error, debug
944	- */
945	- private function logPendingDocumentInfoStatus($documentId, $message, $level)
946	- {
947	- $this->updatePendingDocumentStatus($documentId, $message, $level);
948	- global $default;
949	-
950	- switch ($level)
951	- {
952	- case 'debug':
953	- if ($this->debug)
954	- {
955	- $default->log->debug($message);
956	- }
957	- break;
958	- default:
959	- $default->log->$level($message);
960	- }
961	- }
962	-
963	-
964	-
965	- public function getExtractor($extractorClass)
966	- {
967	- if (empty($extractorClass))
968	- {
969	- return null;
970	- }
971	-
972	- $includeFile = SEARCH2_INDEXER_DIR . 'extractors/' . $extractorClass . '.inc.php';
973	- if (!file_exists($includeFile))
974	- {
975	- throw new Exception("Extractor file does not exist: $includeFile");
976	- }
977	-
978	- require_once($includeFile);
979	-
980	- if (!class_exists($extractorClass))
981	- {
982	- throw new Exception("Extractor '$classname' not defined in file: $includeFile");
983	- }
984	-
985	- $extractor = new $extractorClass();
986	-
987	- if (!($extractor instanceof DocumentExtractor))
988	- {
989	- throw new Exception("Class $classname was expected to be of type DocumentExtractor");
990	- }
991	-
992	- return $extractor;
993	- }
994	-
995	- public static function getIndexingQueue($problemItemsOnly=true)
996	- {
997	-
998	- if ($problemItemsOnly)
999	- {
1000	- $sql = "SELECT
1001	- iff.document_id, iff.indexdate, mt.filetypes, mt.mimetypes, me.name as extractor, iff.what, iff.status_msg, dcv.filename
1002	- FROM
1003	- index_files iff
1004	- INNER JOIN documents d ON iff.document_id=d.id
1005	- INNER JOIN document_metadata_version dmv ON d.metadata_version_id=dmv.id
1006	- INNER JOIN document_content_version dcv ON dmv.content_version_id=dcv.id
1007	- INNER JOIN mime_types mt ON dcv.mime_id=mt.id
1008	- LEFT JOIN mime_extractors me ON mt.extractor_id=me.id
1009	- WHERE
1010	- (iff.status_msg IS NOT NULL AND iff.status_msg <> '') AND d.status_id=1
1011	- ORDER BY indexdate ";
1012	- }
1013	- else
1014	- {
1015	- $sql = "SELECT
1016	- iff.document_id, iff.indexdate, mt.filetypes, mt.mimetypes, me.name as extractor, iff.what, iff.status_msg, dcv.filename
1017	- FROM
1018	- index_files iff
1019	- INNER JOIN documents d ON iff.document_id=d.id
1020	- INNER JOIN document_metadata_version dmv ON d.metadata_version_id=dmv.id
1021	- INNER JOIN document_content_version dcv ON dmv.content_version_id=dcv.id
1022	- INNER JOIN mime_types mt ON dcv.mime_id=mt.id
1023	- LEFT JOIN mime_extractors me ON mt.extractor_id=me.id
1024	- WHERE
1025	- (iff.status_msg IS NULL or iff.status_msg = '') AND d.status_id=1
1026	- ORDER BY indexdate ";
1027	- }
1028	- $aResult = DBUtil::getResultArray($sql);
1029	-
1030	- return $aResult;
1031	- }
1032	-
1033	- public static function getPendingIndexingQueue()
1034	- {
1035	- return Indexer::getIndexingQueue(false);
1036	- }
1037	-
1038	- public function updateIndexStats()
1039	- {
1040	- $optimisationDate = KTUtil::getSystemSetting('luceneOptimisationDate', '');
1041	-
1042	- $noOptimisation = false;
1043	- if ($optimisationDate == '')
1044	- {
1045	- $optimisationDate = _kt('N/A');
1046	- $optimisationPeriod = $optimisationDate;
1047	- }
1048	- else
1049	- {
1050	- $optimisationPeriod = KTUtil::computePeriodToDate($optimisationDate, null, true);
1051	- $noOptimisation = $optimisationPeriod['days'] > 2;
1052	- $optimisationPeriod = $optimisationPeriod['str'];
1053	- $optimisationDate = date('Y-m-d H:i:s', $optimisationDate);
1054	- }
1055	-
1056	- $indexingDate = KTUtil::getSystemSetting('luceneIndexingDate', '');
1057	- if ($indexingDate == '')
1058	- {
1059	- $indexingDate = _kt('N/A');
1060	- $indexingPeriod = $indexingDate;
1061	- }
1062	- else
1063	- {
1064	- $indexingPeriod = KTUtil::computePeriodToDate($indexingDate);
1065	- $indexingDate = date('Y-m-d H:i:s', $indexingDate);
1066	- }
1067	-
1068	- $index = Indexer::get();
1069	- $docsInIndex = $index->getDocumentsInIndex();
1070	-
1071	- // we are only interested in documents that are active
1072	- $sql = "SELECT count(*) as docsInQueue FROM index_files i inner join documents d on i.document_id = d.id where d.status_id=1";
1073	- $docsInQueue = DBUtil::getOneResultKey($sql, 'docsInQueue');
1074	-
1075	- $sql = "SELECT count(*) as errorsInQueue FROM index_files i inner join documents d on i.document_id = d.id where (i.status_msg is not null or i.status_msg <> '') and d.status_id=1";
1076	- $errorsInQueue = DBUtil::getOneResultKey($sql, 'errorsInQueue');
1077	-
1078	- $sql = "SELECT count(*) as docsInRepository FROM documents where status_id=1";
1079	- $docsInRepository = DBUtil::getOneResultKey($sql, 'docsInRepository');
1080	-
1081	- if ($docsInRepository == 0)
1082	- {
1083	- $indexingCoverage = '0.00%';
1084	- $queueCoverage = $indexingCoverage;
1085	- }
1086	- else
1087	- {
1088	- // compute indexing coverage
1089	- $indexingCoverage = _kt('Not Available');
1090	- if (is_numeric($docsInIndex))
1091	- {
1092	- $indexingCoverage = ($docsInIndex * 100) / $docsInRepository;
1093	- $indexingCoverage = number_format($indexingCoverage, 2, '.',',') . '%';
1094	- }
1095	-
1096	- // compute queue coverage
1097	- $queueCoverage = _kt('Not Available');
1098	- if (is_numeric($docsInQueue))
1099	- {
1100	- $queueCoverage = ($docsInQueue * 100) / $docsInRepository;
1101	- $queueCoverage = number_format($queueCoverage, 2, '.',',') . '%';
1102	- }
1103	- }
1104	-
1105	-
1106	- $stats = array(
1107	- 'optimisationDate'=>$optimisationDate,
1108	- 'optimisationPeriod'=>$optimisationPeriod,
1109	- 'indexingDate'=>$indexingDate,
1110	- 'indexingPeriod'=>$indexingPeriod,
1111	- 'docsInIndex'=>$docsInIndex,
1112	- 'docsInQueue'=>$docsInQueue,
1113	- 'errorsInQueue'=>$errorsInQueue,
1114	- 'docsInRepository'=>$docsInRepository,
1115	- 'indexingCoverage'=>$indexingCoverage,
1116	- 'queueCoverage'=>$queueCoverage,
1117	- 'noOptimisation'=>$noOptimisation
1118	- );
1119	-
1120	- KTUtil::setSystemSetting('indexerStats', serialize($stats));
1121	-
1122	- $indexer = Indexer::get();
1123	-
1124	- $diagnosis = $indexer->diagnose();
1125	- KTUtil::setSystemSetting('indexerDiagnostics', serialize($diagnosis));
1126	-
1127	- $extractorDiagnosis = $indexer->diagnoseExtractors();
1128	-
1129	- KTUtil::setSystemSetting('extractorDiagnostics', serialize($extractorDiagnosis));
1130	- }
1131	-
1132	- /**
1133	- * The main function that may be called repeatedly to index documents.
1134	- *
1135	- * @param int $max Default 20
1136	- */
1137	- public function indexDocuments($max=null)
1138	- {
1139	- global $default;
1140	- $config =& KTConfig::getSingleton();
1141	-
1142	- /*$indexLockFile = $config->get('cache/cacheDirectory') . '/main.index.lock';
1143	- if (is_file($indexLockFile))
1144	- {
1145	- $default->log->info('indexDocuments: main.index.lock seems to exist. it could be that the indexing is still underway.');
1146	- $default->log->info('indexDocuments: Remove "' . $indexLockFile . '" if the indexing is not running or extend the frequency at which the background task runs!');
1147	- return;
1148	- }
1149	- touch($indexLockFile);*/
1150	-
1151	-
1152	- $this->checkForRegisteredTypes();
1153	-
1154	- if ($this->debug) $default->log->debug('indexDocuments: start');
1155	- if (!$this->doesDiagnosticsPass())
1156	- {
1157	- //unlink($indexLockFile);
1158	- if ($this->debug) $default->log->debug('indexDocuments: stopping - diagnostics problem. The dashboard will provide more information.');
1159	- return;
1160	- }
1161	-
1162	- if (is_null($max))
1163	- {
1164	- $max = $config->get('indexer/batchDocuments',20);
1165	- }
1166	-
1167	- $this->loadExtractorHooks();
1168	-
1169	- Indexer::clearoutDeleted();
1170	-
1171	- $date = date('Y-m-d H:i:s');
1172	- // identify the indexers that must run
1173	- // mysql specific limit!
1174	- $sql = "SELECT
1175	- iff.document_id, mt.filetypes, mt.mimetypes, me.name as extractor, iff.what
1176	- FROM
1177	- index_files iff
1178	- INNER JOIN documents d ON iff.document_id=d.id
1179	- INNER JOIN document_metadata_version dmv ON d.metadata_version_id=dmv.id
1180	- INNER JOIN document_content_version dcv ON dmv.content_version_id=dcv.id
1181	- INNER JOIN mime_types mt ON dcv.mime_id=mt.id
1182	- LEFT JOIN mime_extractors me ON mt.extractor_id=me.id
1183	- WHERE
1184	- (iff.processdate IS NULL or iff.processdate < date_sub('$date', interval 1 day)) AND dmv.status_id=1
1185	- ORDER BY indexdate
1186	- LIMIT $max";
1187	- $result = DBUtil::getResultArray($sql);
1188	- if (PEAR::isError($result))
1189	- {
1190	- //unlink($indexLockFile);
1191	- if ($this->debug) $default->log->debug('indexDocuments: stopping - db error');
1192	- return;
1193	- }
1194	- KTUtil::setSystemSetting('luceneIndexingDate', time());
1195	-
1196	- // bail if no work to do
1197	- if (count($result) == 0)
1198	- {
1199	- //unlink($indexLockFile);
1200	- if ($this->debug) $default->log->debug('indexDocuments: stopping - no work to be done');
1201	- return;
1202	- }
1203	-
1204	- // identify any documents that need indexing and mark them
1205	- // so they are not taken in a followup run
1206	- $ids = array();
1207	- foreach($result as $docinfo)
1208	- {
1209	- $ids[] = $docinfo['document_id'];
1210	- }
1211	-
1212	- // mark the documents as being processed
1213	-
1214	- $ids=implode(',',$ids);
1215	- $sql = "UPDATE index_files SET processdate='$date' WHERE document_id in ($ids)";
1216	- DBUtil::runQuery($sql);
1217	-
1218	- $extractorCache = array();
1219	- $storageManager = KTStorageManagerUtil::getSingleton();
1220	-
1221	- $tempPath = $config->get("urls/tmpDirectory");
1222	-
1223	- foreach($result as $docinfo)
1224	- {
1225	- // increment indexed documents count
1226	- Indexer::incrementCount();
1227	-
1228	- $docId=$docinfo['document_id'];
1229	- $extension=$docinfo['filetypes'];
1230	- $mimeType=$docinfo['mimetypes'];
1231	- $extractorClass=$docinfo['extractor'];
1232	- $indexDocument = in_array($docinfo['what'], array('A','C'));
1233	- $indexDiscussion = in_array($docinfo['what'], array('A','D'));
1234	- $this->indexingHistory = '';
1235	-
1236	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Indexing docid: %d extension: '%s' mimetype: '%s' extractor: '%s'"), $docId, $extension,$mimeType,$extractorClass), 'debug');
1237	-
1238	- if (empty($extractorClass))
1239	- {
1240	- /*
1241	-
1242	- if no extractor is found and we don't need to index discussions, then we can remove the item from the queue.
1243	-
1244	- */
1245	- if ($indexDiscussion)
1246	- {
1247	- $indexDocument = false;
1248	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Not indexing docid: %d content because extractor could not be resolve. Still indexing discussion."), $docId), 'info');
1249	- }
1250	- else
1251	- {
1252	- Indexer::unqueueDocument($docId, sprintf(_kt("No extractor for docid: %d"),$docId));
1253	- continue;
1254	- }
1255	- }
1256	- else
1257	- {
1258	- /*
1259	-
1260	- If an extractor is available, we must ensure it is enabled.
1261	-
1262	- */
1263	-
1264	- if (!$this->isExtractorEnabled($extractorClass))
1265	- {
1266	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info');
1267	- continue;
1268	- }
1269	- }
1270	-
1271	- if ($this->debug)
1272	- {
1273	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Processing docid: %d.\n"),$docId), 'info');
1274	- }
1275	-
1276	- $document = Document::get($docId);
1277	- if (PEAR::isError($document))
1278	- {
1279	- Indexer::unqueueDocument($docId,sprintf(_kt("indexDocuments: Cannot resolve document id %d: %s."),$docId, $document->getMessage()), 'error');
1280	- continue;
1281	- }
1282	-
1283	- if ($this->restartCurrentBatch)
1284	- {
1285	- Indexer::unqueueDocument($docId);
1286	- Indexer::index($docId, 'A');
1287	- continue;
1288	- }
1289	-
1290	-
1291	- $filename = $document->getFileName();
1292	- if (substr($filename,0,1) == '~' \|\| substr($filename,-1) == '~')
1293	- {
1294	- Indexer::unqueueDocument($docId,sprintf(_kt("indexDocuments: Filename for document id %d starts with a tilde (~). This is assumed to be a temporary file. This is ignored."),$docId), 'error');
1295	- continue;
1296	- }
1297	-
1298	- $removeFromQueue = true;
1299	- if ($indexDocument)
1300	- {
1301	- if (array_key_exists($extractorClass, $extractorCache))
1302	- {
1303	- $extractor = $extractorCache[$extractorClass];
1304	- }
1305	- else
1306	- {
1307	- $extractor = $extractorCache[$extractorClass] = $this->getExtractor($extractorClass);
1308	- }
1309	-
1310	- if (!($extractor instanceof DocumentExtractor))
1311	- {
1312	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("indexDocuments: extractor '%s' is not a document extractor class."),$extractorClass), 'error');
1313	- continue;
1314	- }
1315	-
1316	-
1317	-
1318	- $version = $document->getMajorVersionNumber() . '.' . $document->getMinorVersionNumber();
1319	- $sourceFile = $storageManager->temporaryFile($document);
1320	-
1321	- if (empty($sourceFile) \|\| !is_file($sourceFile))
1322	- {
1323	- Indexer::unqueueDocument($docId,sprintf(_kt("indexDocuments: source file '%s' for document %d does not exist."),$sourceFile,$docId), 'error');
1324	- continue;
1325	- }
1326	-
1327	- if ($extractor->needsIntermediateSourceFile())
1328	- {
1329	- //$extension = pathinfo($document->getFileName(), PATHINFO_EXTENSION);
1330	-
1331	- $intermediate = $tempPath . '/'. $docId . '.' . $extension;
1332	- $result = @copy($sourceFile, $intermediate);
1333	- if ($result === false)
1334	- {
1335	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Could not create intermediate file from document %d"),$docId), 'error');
1336	- // problem. lets try again later. probably permission related. log the issue.
1337	- continue;
1338	- }
1339	- $sourceFile = $intermediate;
1340	- }
1341	-
1342	- $targetFile = tempnam($tempPath, 'ktindexer');
1343	-
1344	- $extractor->setSourceFile($sourceFile);
1345	- $extractor->setMimeType($mimeType);
1346	- $extractor->setExtension($extension);
1347	- $extractor->setTargetFile($targetFile);
1348	- $extractor->setDocument($document);
1349	- $extractor->setIndexingStatus(null);
1350	- $extractor->setExtractionStatus(null);
1351	-
1352	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Extra Info docid: %d Source File: '%s' Target File: '%s'"),$docId,$sourceFile,$targetFile), 'debug');
1353	-
1354	- $this->executeHook($extractor, 'pre_extract');
1355	- $this->executeHook($extractor, 'pre_extract', $mimeType);
1356	- $removeFromQueue = false;
1357	-
1358	- if ($extractor->extractTextContent())
1359	- {
1360	- // the extractor may need to create another target file
1361	- $targetFile = $extractor->getTargetFile();
1362	-
1363	- $extractor->setExtractionStatus(true);
1364	- $this->executeHook($extractor, 'pre_index');
1365	- $this->executeHook($extractor, 'pre_index', $mimeType);
1366	-
1367	- $title = $document->getName();
1368	- if ($indexDiscussion)
1369	- {
1370	- if (!$this->filterText($targetFile))
1371	- {
1372	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"),$docId), 'error');
1373	- }
1374	- else
1375	- {
1376	- $indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version);
1377	- $removeFromQueue = $indexStatus;
1378	- if (!$indexStatus)
1379	- {
1380	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId), 'error');
1381	- }
1382	-
1383	- $extractor->setIndexingStatus($indexStatus);
1384	- }
1385	- }
1386	- else
1387	- {
1388	- if (!$this->filterText($targetFile))
1389	- {
1390	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"),$docId), 'error');
1391	- }
1392	- else
1393	- {
1394	- $indexStatus = $this->indexDocument($docId, $targetFile, $title, $version);
1395	- $removeFromQueue = $indexStatus;
1396	-
1397	- if (!$indexStatus)
1398	- {
1399	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocument"),$docId), 'error');
1400	- $this->logPendingDocumentInfoStatus($docId, '<output>' . $extractor->output . '</output>', 'error');
1401	- }
1402	-
1403	- $extractor->setIndexingStatus($indexStatus);
1404	- }
1405	- }
1406	-
1407	- $this->executeHook($extractor, 'post_index', $mimeType);
1408	- $this->executeHook($extractor, 'post_index');
1409	- }
1410	- else
1411	- {
1412	- $extractor->setExtractionStatus(false);
1413	- $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Could not extract contents from document %d"),$docId), 'error');
1414	- $this->logPendingDocumentInfoStatus($docId, '<output>' . $extractor->output . '</output>', 'error');
1415	- }
1416	-
1417	- $this->executeHook($extractor, 'post_extract', $mimeType);
1418	- $this->executeHook($extractor, 'post_extract');
1419	-
1420	- if ($extractor->needsIntermediateSourceFile())
1421	- {
1422	- @unlink($sourceFile);
1423	- }
1424	-
1425	- @unlink($targetFile);
1426	-
1427	- }
1428	- else
1429	- {
1430	- $indexStatus = $this->indexDiscussion($docId);
1431	- $removeFromQueue = $indexStatus;
1432	- }
1433	-
1434	- if ($removeFromQueue)
1435	- {
1436	- Indexer::unqueueDocument($docId, sprintf(_kt("Done indexing docid: %d"),$docId));
1437	- }
1438	- else
1439	- {
1440	- if ($this->debug) $default->log->debug(sprintf(_kt("Document docid: %d was not removed from the queue as it looks like there was a problem with the extraction process"),$docId));
1441	- }
1442	- }
1443	- if ($this->debug) $default->log->debug('indexDocuments: done');
1444	- //unlink($indexLockFile);
1445	- }
1446	-
1447	- public function migrateDocuments($max=null)
1448	- {
1449	- global $default;
1450	-
1451	- $default->log->info(_kt('migrateDocuments: starting'));
1452	-
1453	- if (!$this->doesDiagnosticsPass(true))
1454	- {
1455	- $default->log->info(_kt('migrateDocuments: stopping - diagnostics problem. The dashboard will provide more information.'));
1456	- return;
1457	- }
1458	-
1459	- if (KTUtil::getSystemSetting('migrationComplete') == 'true')
1460	- {
1461	- $default->log->info(_kt('migrateDocuments: stopping - migration is complete.'));
1462	- return;
1463	- }
1464	-
1465	- $config =& KTConfig::getSingleton();
1466	- if (is_null($max))
1467	- {
1468	- $max = $config->get('indexer/batchMigrateDocument',500);
1469	- }
1470	-
1471	- $lockFile = $config->get('cache/cacheDirectory') . '/migration.lock';
1472	- if (is_file($lockFile))
1473	- {
1474	- $default->log->info(_kt('migrateDocuments: stopping - migration lockfile detected.'));
1475	- return;
1476	- }
1477	- touch($lockFile);
1478	-
1479	- $startTime = KTUtil::getSystemSetting('migrationStarted');
1480	- if (is_null($startTime))
1481	- {
1482	- KTUtil::setSystemSetting('migrationStarted', time());
1483	- }
1484	-
1485	- $maxLoops = 5;
1486	-
1487	- $max = ceil($max / $maxLoops);
1488	-
1489	- $start =KTUtil::getBenchmarkTime();
1490	- $noDocs = false;
1491	- $numDocs = 0;
1492	-
1493	- for($loop=0;$loop<$maxLoops;$loop++)
1494	- {
1495	-
1496	- $sql = "SELECT
1497	- document_id, document_text
1498	- FROM
1499	- document_text
1500	- ORDER BY document_id
1501	- LIMIT $max";
1502	- $result = DBUtil::getResultArray($sql);
1503	- if (PEAR::isError($result))
1504	- {
1505	- $default->log->info(_kt('migrateDocuments: db error'));
1506	- break;
1507	- }
1508	-
1509	- $docs = count($result);
1510	- if ($docs == 0)
1511	- {
1512	- $noDocs = true;
1513	- break;
1514	- }
1515	- $numDocs += $docs;
1516	-
1517	- foreach($result as $docinfo)
1518	- {
1519	- $docId = $docinfo['document_id'];
1520	-
1521	- $document = Document::get($docId);
1522	- if (PEAR::isError($document) \|\| is_null($document))
1523	- {
1524	- $sql = "DELETE FROM document_text WHERE document_id=$docId";
1525	- DBUtil::runQuery($sql);
1526	- $default->log->error(sprintf(_kt('migrateDocuments: Could not get document %d\'s document! Removing content!'),$docId));
1527	- continue;
1528	- }
1529	-
1530	- $version = $document->getMajorVersionNumber() . '.' . $document->getMinorVersionNumber();
1531	-
1532	- $targetFile = tempnam($tempPath, 'ktindexer');
1533	-
1534	- if (file_put_contents($targetFile, $docinfo['document_text']) === false)
1535	- {
1536	- $default->log->error(sprintf(_kt('migrateDocuments: Cannot write to \'%s\' for document id %d'), $targetFile, $docId));
1537	- continue;
1538	- }
1539	- // free memory asap ;)
1540	- unset($docinfo['document_text']);
1541	-
1542	- $title = $document->getName();
1543	-
1544	- $indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version);
1545	-
1546	- if ($indexStatus)
1547	- {
1548	- $sql = "DELETE FROM document_text WHERE document_id=$docId";
1549	- DBUtil::runQuery($sql);
1550	- }
1551	- else
1552	- {
1553	- $default->log->error(sprintf(_kt("migrateDocuments: Problem indexing document %d"), $docId));
1554	- }
1555	-
1556	- @unlink($targetFile);
1557	- }
1558	- }
1559	-
1560	- @unlink($lockFile);
1561	-
1562	- $time = KTUtil::getBenchmarkTime() - $start;
1563	-
1564	- KTUtil::setSystemSetting('migrationTime', KTUtil::getSystemSetting('migrationTime',0) + $time);
1565	- KTUtil::setSystemSetting('migratedDocuments', KTUtil::getSystemSetting('migratedDocuments',0) + $numDocs);
1566	-
1567	- $default->log->info(sprintf(_kt('migrateDocuments: stopping - done in %d seconds!'), $time));
1568	- if ($noDocs)
1569	- {
1570	- $default->log->info(_kt('migrateDocuments: Completed!'));
1571	- KTUtil::setSystemSetting('migrationComplete', 'true');
1572	- schedulerUtil::deleteByName('Index Migration');
1573	- $default->log->debug(_kt('migrateDocuments: Disabling \'Index Migration\' task by removing scheduler entry.'));
1574	- }
1575	- }
1576	-
1577	- /**
1578	- * Index a document. The base class must override this function.
1579	- *
1580	- * @param int $docId
1581	- * @param string $textFile
1582	- */
1583	- protected abstract function indexDocument($docId, $textFile, $title, $version);
1584	-
1585	-
1586	- public function updateDocumentIndex($docId, $text)
1587	- {
1588	- $config = KTConfig::getSingleton();
1589	- $tempPath = $config->get("urls/tmpDirectory");
1590	- $tempFile = tempnam($tempPath,'ud_');
1591	-
1592	- file_put_contents($tempFile, $text);
1593	-
1594	- $document = Document::get($docId);
1595	- $title = $document->getDescription();
1596	- $version = $document->getVersion();
1597	-
1598	- $result = $this->indexDocument($docId, $tempFile, $title, $version);
1599	-
1600	- if (file_exists($tempFile))
1601	- {
1602	- unlink($tempFile);
1603	- }
1604	-
1605	- return $result;
1606	- }
1607	-
1608	- /**
1609	- * Index a discussion. The base class must override this function.
1610	- *
1611	- * @param int $docId
1612	- */
1613	- protected abstract function indexDiscussion($docId);
1614	-
1615	- /**
1616	- * Diagnose the indexer. e.g. Check that the indexing server is running.
1617	- *
1618	- */
1619	- public abstract function diagnose();
1620	-
1621	- /**
1622	- * Diagnose the extractors.
1623	- *
1624	- * @return array
1625	- */
1626	- public function diagnoseExtractors()
1627	- {
1628	- $diagnosis = $this->_diagnose($this->extractorPath, 'DocumentExtractor', 'Extractor.inc.php');
1629	- $diagnosis = array_merge($diagnosis, $this->_diagnose($this->hookPath, 'Hook', 'Hook.inc.php'));
1630	-
1631	- return $diagnosis;
1632	- }
1633	-
1634	- /**
1635	- * This is a refactored diagnose function.
1636	- *
1637	- * @param string $path
1638	- * @param string $class
1639	- * @param string $extension
1640	- * @return array
1641	- */
1642	- private function _diagnose($path, $baseclass, $extension)
1643	- {
1644	- global $default;
1645	-
1646	- $diagnoses = array();
1647	-
1648	- $dir = opendir(SearchHelper::correctPath($path));
1649	- $extlen = - strlen($extension);
1650	-
1651	- while (($file = readdir($dir)) !== false)
1652	- {
1653	- if (substr($file,0,1) == '.')
1654	- {
1655	- continue;
1656	- }
1657	- if (substr($file,$extlen) != $extension)
1658	- {
1659	- $default->log->error(sprintf(_kt("diagnose: '%s' does not have extension '%s'."), $file, $extension));
1660	- continue;
1661	- }
1662	-
1663	- require_once($path . '/' . $file);
1664	-
1665	- $class = substr($file, 0, -8);
1666	- if (!class_exists($class))
1667	- {
1668	- $default->log->error(sprintf(_kt("diagnose: class '%s' does not exist."), $class));
1669	- continue;
1670	- }
1671	-
1672	- if (!$this->isExtractorEnabled($class))
1673	- {
1674	- $default->log->debug(sprintf(_kt("diagnose: extractor '%s' is disabled."), $class));
1675	- continue;
1676	- }
1677	-
1678	- $extractor = new $class();
1679	- if (!is_a($extractor, $baseclass))
1680	- {
1681	- $default->log->error(sprintf(_kt("diagnose(): '%s' is not of type DocumentExtractor"), $class));
1682	- continue;
1683	- }
1684	-
1685	- $types = $extractor->getSupportedMimeTypes();
1686	- if (empty($types))
1687	- {
1688	- if ($this->debug) $default->log->debug(sprintf(_kt("diagnose: class '%s' does not support any types."), $class));
1689	- continue;
1690	- }
1691	-
1692	- $diagnosis=$extractor->diagnose();
1693	- if (empty($diagnosis))
1694	- {
1695	- continue;
1696	- }
1697	- $diagnoses[$class] = array(
1698	- 'name'=>$extractor->getDisplayName(),
1699	- 'diagnosis'=>$diagnosis
1700	- );
1701	-
1702	- }
1703	- closedir($dir);
1704	-
1705	- return $diagnoses;
1706	- }
1707	-
1708	-
1709	- /**
1710	- * Register the extractor types.
1711	- *
1712	- * @param boolean $clear. Optional. Defaults to false.
1713	- */
1714	- public function registerTypes($clear=false)
1715	- {
1716	- if ($clear)
1717	- {
1718	- $this->clearExtractors();
1719	- }
1720	- $dir = opendir(SearchHelper::correctPath($this->extractorPath));
1721	- while (($file = readdir($dir)) !== false)
1722	- {
1723	- if (substr($file,-17) == 'Extractor.inc.php')
1724	- {
1725	- require_once($this->extractorPath . '/' . $file);
1726	- $class = substr($file, 0, -8);
1727	-
1728	- if (!class_exists($class))
1729	- {
1730	- // if the class does not exist, we can't do anything.
1731	- continue;
1732	- }
1733	-
1734	- $extractor = new $class;
1735	- if ($extractor instanceof DocumentExtractor)
1736	- {
1737	- $extractor->registerMimeTypes();
1738	- }
1739	- }
1740	- }
1741	- closedir($dir);
1742	- }
1743	-
1744	- /**
1745	- * This is used as a possible obtimisation effort. It may be overridden in that case.
1746	- *
1747	- * @param int $docId
1748	- * @param string $textFile
1749	- */
1750	- protected function indexDocumentAndDiscussion($docId, $textFile, $title, $version)
1751	- {
1752	- $this->indexDocument($docId, $textFile, $title, $version);
1753	- $this->indexDiscussion($docId);
1754	- }
1755	-
1756	- /**
1757	- * Remove the document from the queue. This is normally called when it has been processed.
1758	- *
1759	- * @param int $docid
1760	- */
1761	- public static function unqueueDocument($docid, $reason=false, $level='debug')
1762	- {
1763	- $sql = "DELETE FROM index_files WHERE document_id=$docid";
1764	- DBUtil::runQuery($sql);
1765	- if ($reason !== false)
1766	- {
1767	- global $default;
1768	- $default->log->$level("Indexer: removing document $docid from the queue - $reason");
1769	- }
1770	- }
1771	-
1772	- /**
1773	- * Run a query on the index.
1774	- *
1775	- * @param string $query
1776	- * @return array
1777	- */
1778	- public abstract function query($query);
1779	-
1780	- /**
1781	- * Converts an integer to a string that can be easily compared and reversed.
1782	- *
1783	- * @param int $int
1784	- * @return string
1785	- */
1786	- public static function longToString($int)
1787	- {
1788	- $maxlen = 14;
1789	-
1790	- $a2z = array('a','b','c','d','e','f','g','h','i','j');
1791	- $o29 = array('0','1','2','3','4','5','6','7','8','9');
1792	- $l = str_pad('',$maxlen - strlen("$int"),'0') . $int;
1793	-
1794	- return str_replace($o29, $a2z, $l);
1795	- }
1796	-
1797	- /**
1798	- * Converts a string to an integer.
1799	- *
1800	- * @param string $str
1801	- * @return int
1802	- */
1803	- public static function stringToLong($str)
1804	- {
1805	- $a2z = array('a','b','c','d','e','f','g','h','i','j');
1806	- $o29 = array('0','1','2','3','4','5','6','7','8','9');
1807	-
1808	- $int = str_replace($a2z, $o29, $str) + 0;
1809	-
1810	- return $int;
1811	- }
1812	-
1813	- /**
1814	- * Possibly we can optimise indexes. This method must be overriden.
1815	- * The new function must call the parent!
1816	- *
1817	- */
1818	- public function optimise()
1819	- {
1820	- KTUtil::setSystemSetting('luceneOptimisationDate', time());
1821	- }
1822	-
1823	- /**
1824	- * Shuts down the indexer
1825	- *
1826	- */
1827	- public function shutdown()
1828	- {
1829	- // do nothing generally
1830	- }
1831	-
1832	- /**
1833	- * Returns the name of the indexer.
1834	- *
1835	- * @return string
1836	- */
1837	- public abstract function getDisplayName();
1838	-
1839	-
1840	- /**
1841	- * Returns the number of non-deleted documents in the index.
1842	- *
1843	- * @return int
1844	- */
1845	- public abstract function getDocumentsInIndex();
1846	-
1847	- public abstract function isDocumentIndexed($documentId);
1848	-
1849	- /**
1850	- * Returns the path to the index directory
1851	- *
1852	- * @return string
1853	- */
1854	- public function getIndexDirectory()
1855	- {
1856	- $config = KTConfig::getSingleton();
1857	- $directory = $config->get('indexer/luceneDirectory');
1858	- return $directory;
1859	- }
1860	-}
1861	-
1862	-?>	1	+<?php
		2	+
		3	+/**
		4	+ * $Id:$
		5	+ *
		6	+ * KnowledgeTree Community Edition
		7	+ * Document Management Made Simple
		8	+ * Copyright (C) 2008 KnowledgeTree Inc.
		9	+ * Portions copyright The Jam Warehouse Software (Pty) Limited
		10	+ *
		11	+ * This program is free software; you can redistribute it and/or modify it under
		12	+ * the terms of the GNU General Public License version 3 as published by the
		13	+ * Free Software Foundation.
		14	+ *
		15	+ * This program is distributed in the hope that it will be useful, but WITHOUT
		16	+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
		17	+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
		18	+ * details.
		19	+ *
		20	+ * You should have received a copy of the GNU General Public License
		21	+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
		22	+ *
		23	+ * You can contact KnowledgeTree Inc., PO Box 7775 #87847, San Francisco,
		24	+ * California 94120-7775, or email info@knowledgetree.com.
		25	+ *
		26	+ * The interactive user interfaces in modified source and object code versions
		27	+ * of this program must display Appropriate Legal Notices, as required under
		28	+ * Section 5 of the GNU General Public License version 3.
		29	+ *
		30	+ * In accordance with Section 7(b) of the GNU General Public License version 3,
		31	+ * these Appropriate Legal Notices must retain the display of the "Powered by
		32	+ * KnowledgeTree" logo and retain the original copyright notice. If the display of the
		33	+ * logo is not reasonably feasible for technical reasons, the Appropriate Legal Notices
		34	+ * must display the words "Powered by KnowledgeTree" and retain the original
		35	+ * copyright notice.
		36	+ * Contributor( s): ______________________________________
		37	+ *
		38	+ */
		39	+
		40	+define('SEARCH2_INDEXER_DIR',realpath(dirname(__FILE__)) . '/');
		41	+require_once('indexing/extractorCore.inc.php');
		42	+require_once(KT_DIR . '/plugins/ktcore/scheduler/schedulerUtil.php');
		43	+require_once(KT_DIR . '/ktapi/ktapi.inc.php');
		44	+
		45	+class IndexerInconsistencyException extends Exception {};
		46	+
		47	+// TODO: Query Result Items code should be moved into the Search section. It has less to do with indexing...
		48	+
		49	+class QueryResultItem
		50	+{
		51	+ protected $id;
		52	+ protected $title;
		53	+ protected $rank;
		54	+ protected $text;
		55	+ protected $fullpath;
		56	+
		57	+ public function __construct($id, $title, $rank, $text, $fullpath)
		58	+ {
		59	+ $this->id = $id;
		60	+ $this->title = $title;
		61	+ $this->rank = $rank;
		62	+ $this->text = $text;
		63	+ $this->fullpath = $fullpath;
		64	+ }
		65	+
		66	+ public function getId() { return $this->id; }
		67	+ public function getRealId() { return $this->id; }
		68	+
		69	+ public function getIsProxy() { return $this instanceof ProxyResultItem; }
		70	+ public function getIsFolder() { return substr(get_class($this), 0, 6) == 'Folder' ; }
		71	+ public function getIsDocument() { return substr(get_class($this), 0, 8) == 'Document' ; }
		72	+
		73	+ public function setRank($value)
		74	+ {
		75	+ $this->rank = number_format($value,2,'.',',');
		76	+ }
		77	+
		78	+ public function getIsLive()
		79	+ {
		80	+ return true;
		81	+ }
		82	+
		83	+ public function setTitle($value)
		84	+ {
		85	+ $this->title = $value;
		86	+ }
		87	+
		88	+ public function setText($value)
		89	+ {
		90	+ $this->text = $value;
		91	+ }
		92	+
		93	+ public function getRelevance() { return (float) $this->rank; }
		94	+ public function getRank() { return $this->getRelevance(); }
		95	+ public function getText() { return (string) $this->text; }
		96	+ public function getTitle() { return (string) $this->title; }
		97	+ public function getFullPath() { return (string) $this->fullpath; }
		98	+
		99	+ protected function __get($property)
		100	+ {
		101	+ if (empty($property))
		102	+ {
		103	+ return '';
		104	+ }
		105	+
		106	+ $method = 'get' . $property;
		107	+ if (method_exists($this, $method))
		108	+ {
		109	+ return $this->$method();
		110	+ }
		111	+ return $this->getUnknown();
		112	+ }
		113	+
		114	+ protected function getUnknown()
		115	+ {
		116	+ return _kt('n/a');
		117	+ }
		118	+
		119	+ protected function __set($property, $value)
		120	+ {
		121	+ if (empty($property))
		122	+ {
		123	+ return '';
		124	+ }
		125	+
		126	+ $method = 'set' . $property;
		127	+ if (method_exists($this, $method))
		128	+ {
		129	+ return $this->$method($value);
		130	+ }
		131	+ throw new Exception("Unknown property '$property' to set on QueryResultItem");
		132	+ }
		133	+}
		134	+
		135	+class ProxyResultItem extends QueryResultItem
		136	+{
		137	+ protected $proxy;
		138	+ protected $proxyId;
		139	+
		140	+ public function __construct($proxyId, $proxy)
		141	+ {
		142	+ parent::__construct($proxyId, $proxy->getTitle, $proxy->getRank(), $proxy->getText(), $proxy->getFullPath());
		143	+ $this->proxyId = $proxyId;
		144	+ $this->proxy = $proxy;
		145	+ }
		146	+
		147	+ public function getId() { return $this->proxyId; }
		148	+ public function getTitle() { return $this->proxy->getTitle(); }
		149	+ public function getRealId() { return $this->proxy->getId(); }
		150	+
		151	+ protected function __get($property)
		152	+ {
		153	+ $method = 'get' . $property;
		154	+
		155	+ if (method_exists($this, $method))
		156	+ {
		157	+ return $this->$method();
		158	+ }
		159	+ else
		160	+ {
		161	+ return $this->proxy->$method();
		162	+ }
		163	+ }
		164	+
		165	+ protected function __set($property, $value)
		166	+ {
		167	+ $method = 'set' . $property;
		168	+ if (method_exists($this, $method))
		169	+ {
		170	+ return $this->$method($value);
		171	+ }
		172	+ else
		173	+ {
		174	+ return $this->proxy->$method($value);
		175	+ }
		176	+ }
		177	+}
		178	+
		179	+class DocumentResultItem extends QueryResultItem
		180	+{
		181	+ protected $filesize;
		182	+ protected $live;
		183	+ protected $version;
		184	+ protected $mimeType;
		185	+ protected $filename;
		186	+ protected $thumbnail; // TODO: if not null, gui can display a thumbnail
		187	+ protected $viewer; // TODO: if not null, a viewer can be used to view the document
		188	+ protected $document;
		189	+ protected $checkedOutUser;
		190	+ protected $dateCheckedout;
		191	+ protected $workflowState;
		192	+ protected $workflow;
		193	+ protected $modifiedBy;
		194	+ protected $dateModified;
		195	+ protected $createdBy;
		196	+ protected $dateCreated;
		197	+ protected $owner;
		198	+ protected $immutable;
		199	+ protected $deleted;
		200	+ protected $status;
		201	+ protected $folderId;
		202	+ protected $storagePath;
		203	+ protected $documentType;
		204	+ protected $mimeIconPath;
		205	+ protected $mimeDisplay;
		206	+ protected $oemDocumentNo;
		207	+
		208	+ public function __construct($document_id, $rank=null, $title=null, $text=null, $fullpath = null)
		209	+ {
		210	+ parent::__construct($document_id, $title, $rank, $text, $fullpath);
		211	+ $this->live = true;
		212	+ $this->loadDocumentInfo();
		213	+ }
		214	+
		215	+ // TODO: this is bad. must refactor to do the query on the group of documents.
		216	+ public function loadDocumentInfo()
		217	+ {
		218	+ global $default;
		219	+ $sql = "SELECT
		220	+ d.folder_id, f.full_path, f.name, dcv.size as filesize, dcv.major_version,
		221	+ dcv.minor_version, dcv.filename, cou.name as checkoutuser, w.human_name as workflow, ws.human_name as workflowstate,
		222	+ mt.mimetypes as mimetype, md.mime_doc as mimedoc, d.checkedout, mbu.name as modifiedbyuser, d.modified,
		223	+ cbu.name as createdbyuser, ou.name as owneruser, d.immutable, d.status_id, d.created,dcv.storage_path, dtl.name as document_type,
		224	+ mt.icon_path as mime_icon_path, mt.friendly_name as mime_display, d.oem_no, dmv.name as title
		225	+ FROM
		226	+ documents d
		227	+ INNER JOIN document_metadata_version dmv ON d.metadata_version_id = dmv.id
		228	+ INNER JOIN document_content_version dcv ON dmv.content_version_id = dcv.id
		229	+ INNER JOIN mime_types mt ON dcv.mime_id=mt.id
		230	+ LEFT JOIN document_types_lookup dtl ON dtl.id=dmv.document_type_id
		231	+ LEFT JOIN folders f ON f.id=d.folder_id
		232	+ LEFT JOIN users cou ON d.checked_out_user_id=cou.id
		233	+ LEFT JOIN workflows w ON dmv.workflow_id=w.id
		234	+ LEFT JOIN workflow_states ws ON dmv.workflow_state_id = ws.id
		235	+ LEFT JOIN mime_documents md ON mt.mime_document_id = md.id
		236	+ LEFT JOIN users mbu ON d.modified_user_id=mbu.id
		237	+ LEFT JOIN users cbu ON d.creator_id=cbu.id
		238	+ LEFT JOIN users ou ON d.owner_id=ou.id
		239	+ WHERE
		240	+ d.id=$this->id";
		241	+
		242	+ $result = DBUtil::getOneResult($sql);
		243	+
		244	+ if (PEAR::isError($result) \|\| empty($result))
		245	+ {
		246	+ $this->live = false;
		247	+ if (PEAR::isError($result))
		248	+ {
		249	+ throw new Exception('Database exception! There appears to be an error in the system: ' .$result->getMessage());
		250	+ }
		251	+
		252	+ $default->log->error('QueryResultItem: $result is null');
		253	+ $msg = 'The database did not have a record matching the result from the document indexer. This may occur if there is an inconsistency between the document indexer and the repository. The indexer needs to be repaired.';
		254	+ $default->log->error('QueryResultItem: ' . $msg);
		255	+ // TODO: repair process where we scan documents in index, and delete those for which there is nothing in the repository
		256	+ throw new IndexerInconsistencyException(_kt($msg));
		257	+ }
		258	+
		259	+ // document_id, relevance, text, title
		260	+
		261	+ $this->documentType = $result['document_type'];
		262	+ $this->filename=$result['filename'];
		263	+ $this->filesize = KTUtil::filesizeToString($result['filesize']);
		264	+ $this->folderId = $result['folder_id'];
		265	+ $this->title = $result['title'];
		266	+
		267	+ $this->createdBy = $result['createdbyuser'];
		268	+ $this->dateCreated = $result['created'];
		269	+
		270	+ $this->modifiedBy = $result['modifiedbyuser'];
		271	+ $this->dateModified = $result['modified'];
		272	+
		273	+ $this->checkedOutUser = $result['checkoutuser'];
		274	+ $this->dateCheckedout = $result['checkedout'];
		275	+
		276	+ $this->owner = $result['owneruser'];
		277	+
		278	+ $this->version = $result['major_version'] . '.' . $result['minor_version'];
		279	+
		280	+ $this->immutable = ($result['immutable'] + 0)?_kt('Immutable'):'';
		281	+
		282	+ $this->workflow = $result['workflow'];
		283	+ $this->workflowState = $result['workflowstate'];
		284	+
		285	+ $this->oemDocumentNo = $result['oem_no'];
		286	+ if (empty($this->oemDocumentNo)) $this->oemDocumentNo = 'n/a';
		287	+
		288	+ if (is_null($result['name']))
		289	+ {
		290	+ $this->fullpath = '(orphaned)';
		291	+ }
		292	+ else
		293	+ {
		294	+ $this->fullpath = $result['full_path'];
		295	+ }
		296	+
		297	+ $this->mimeType = $result['mimetype'];
		298	+ $this->mimeIconPath = $result['mime_icon_path'];
		299	+ if (empty($this->mimeIconPath))
		300	+ {
		301	+ $this->mimeIconPath = 'unspecified_type';
		302	+ }
		303	+ $this->mimeDisplay = $result['mime_display'];
		304	+
		305	+ $this->storagePath = $result['storage_path'];
		306	+ $this->status = Document::getStatusString($result['status_id']);
		307	+ }
		308	+
		309	+ public function getDocumentID() { return $this->getId(); }
		310	+ public function getIsLive() { return (bool) $this->live; }
		311	+ public function getFilesize() { return $this->filesize; }
		312	+ public function getVersion() { return (string) $this->version; }
		313	+ public function getFilename() { return (string)$this->filename; }
		314	+ public function getFolderId() { return (int)$this->folderId; }
		315	+ public function getOemDocumentNo() { return (string) $this->oemDocumentNo; }
		316	+ public function getDocument() { return Document::get($this->id); }
		317	+ public function getIsAvailable() { return $this->Document->isLive(); }
		318	+ public function getCheckedOutUser() { return (string) $this->checkedOutUser; }
		319	+ public function getCheckedOutByr() { return $this->getCheckedOutUser(); }
		320	+ public function getWorkflowOnly() { return (string)$this->workflow; }
		321	+ public function getWorkflow() { return $this->getWorkflow(); }
		322	+ public function getWorkflowStateOnly() { return (string)$this->workflowState; }
		323	+ public function getWorkflowState() { return $this->getWorkflowStateOnly(); }
		324	+ public function getWorkflowAndState() {
		325	+ if (is_null($this->workflow))
		326	+ {
		327	+ return '';
		328	+ }
		329	+ return "$this->workflow - $this->workflowState";
		330	+ }
		331	+ public function getMimeType() { return (string) $this->mimeType; }
		332	+ public function getMimeIconPath() { return (string) $this->mimeIconPath; }
		333	+ public function getMimeDisplay() { return (string) $this->mimeDisplay; }
		334	+ public function getDateCheckedOut() { return (string) $this->dateCheckedout; }
		335	+ public function getModifiedBy() { return (string) $this->modifiedBy; }
		336	+ public function getDateModified() { return (string) $this->dateModified; }
		337	+ public function getCreatedBy() { return (string) $this->createdBy; }
		338	+ public function getDateCreated() { return (string) $this->dateCreated; }
		339	+ public function getOwner() { return (string) $this->owner; }
		340	+ public function getOwnedBy() { return $this->getOwner(); }
		341	+ public function getIsImmutable() { return (bool) $this->immutable; }
		342	+ public function getImmutable() { return $this->getIsImmutable(); }
		343	+ public function getStatus() { return $this->status; }
		344	+ public function getStoragePath() { return $this->storagePath; }
		345	+ public function getDocumentType() { return $this->documentType; }
		346	+ public function getPermissions() { return KTAPI_Document::get_permission_string($this->Document); }
		347	+ public function getCanBeReadByUser() {
		348	+ if (!$this->live)
		349	+ return false;
		350	+ if (Permission::userHasDocumentReadPermission($this->Document))
		351	+ return true;
		352	+ if (Permission::adminIsInAdminMode())
		353	+ return true;
		354	+ return false;
		355	+ }
		356	+}
		357	+
		358	+class FolderResultItem extends QueryResultItem
		359	+{
		360	+ protected $folder;
		361	+ protected $createdBy;
		362	+ protected $parentId;
		363	+
		364	+ public function __construct($folder_id, $rank=null, $title=null, $text=null, $fullpath = null)
		365	+ {
		366	+ parent::__construct($folder_id, $title, $rank, $text, $fullpath);
		367	+ $this->loadFolderInfo();
		368	+ }
		369	+
		370	+ public function getFolderID() { return $this->getId(); }
		371	+ public function getParentID() { return $this->parentId; }
		372	+ public function getCreatedBy() { return $this->createdBy; }
		373	+ public function getMimeIconPath() { return 'folder'; }
		374	+ public function getFolder() { return Folder::get($this->getFolderID()); }
		375	+ public function getPermissions() { return KTAPI_Folder::get_permission_string($this->Folder); }
		376	+
		377	+ public function loadFolderInfo()
		378	+ {
		379	+ global $default;
		380	+ $folder = $this->getFolder();
		381	+ if (PEAR::isError($folder))
		382	+ {
		383	+ throw new Exception('Database exception! There appears to be an error in the system: ' .$result->getMessage());
		384	+ }
		385	+ $this->title = $folder->getName();
		386	+ $this->fullpath = '/' . $folder->getFullPath();
		387	+ $this->parentId = $folder->getParentId();
		388	+
		389	+ $user = User::get($folder->getCreatorID());
		390	+ $this->createdBy = (PEAR::isError($user))?_kt('Unknown'):$user->getName();
		391	+ }
		392	+
		393	+}
		394	+
		395	+class DocumentShortcutResultItem extends ProxyResultItem
		396	+{
		397	+ public function getDocumentID() { return $this->getId(); }
		398	+ public function getMimeIconPath() { return $this->proxy->getMimeIconPath() . '_shortcut'; }
		399	+
		400	+}
		401	+
		402	+class FolderShortcutResultItem extends ProxyResultItem
		403	+{
		404	+ public function getFolderID() { return $this->getId(); }
		405	+ public function getMimeIconPath() { return 'folder_shortcut'; }
		406	+
		407	+}
		408	+
		409	+function MatchResultCompare($a, $b)
		410	+{
		411	+ if ($a->Rank == $b->Rank) {
		412	+ return 0;
		413	+ }
		414	+ return ($a->Rank < $b->Rank) ? -1 : 1;
		415	+}
		416	+
		417	+abstract class Indexer
		418	+{
		419	+ /**
		420	+ * Cache of extractors
		421	+ *
		422	+ * @var array
		423	+ */
		424	+ private $extractorCache;
		425	+
		426	+ /**
		427	+ * Indicates if the indexer will do logging.
		428	+ *
		429	+ * @var boolean
		430	+ */
		431	+ private $debug;
		432	+ /**
		433	+ * Cache on mime related hooks
		434	+ *
		435	+ * @var unknown_type
		436	+ */
		437	+ private $mimeHookCache;
		438	+ /**
		439	+ * Cache on general hooks.
		440	+ *
		441	+ * @var array
		442	+ */
		443	+ private $generalHookCache;
		444	+
		445	+ /**
		446	+ * This is a path to the extractors.
		447	+ *
		448	+ * @var string
		449	+ */
		450	+ private $extractorPath;
		451	+ /**
		452	+ * This is a path to the hooks.
		453	+ *
		454	+ * @var string
		455	+ */
		456	+ private $hookPath;
		457	+
		458	+ private $enabledExtractors;
		459	+
		460	+ /**
		461	+ * Initialise the indexer
		462	+ *
		463	+ */
		464	+ protected function __construct()
		465	+ {
		466	+ $config = KTConfig::getSingleton();
		467	+
		468	+ $this->extractorCache = array();
		469	+ $this->debug = $config->get('indexer/debug', true);
		470	+ $this->hookCache = array();
		471	+ $this->generalHookCache = array();
		472	+ $this->extractorPath = $config->get('indexer/extractorPath', 'extractors');
		473	+ $this->hookPath = $config->get('indexer/extractorHookPath','extractorHooks');
		474	+
		475	+ $this->loadExtractorStatus();
		476	+ }
		477	+
		478	+ /**
		479	+ * Get the list if enabled extractors
		480	+ *
		481	+ */
		482	+ private function loadExtractorStatus()
		483	+ {
		484	+ $sql = "SELECT id, name FROM mime_extractors WHERE active=1";
		485	+ $rs = DBUtil::getResultArray($sql);
		486	+ $this->enabledExtractors = array();
		487	+ foreach($rs as $item)
		488	+ {
		489	+ $this->enabledExtractors[] = $item['name'];
		490	+ }
		491	+ }
		492	+
		493	+ private function isExtractorEnabled($extractor)
		494	+ {
		495	+ return in_array($extractor, $this->enabledExtractors);
		496	+ }
		497	+
		498	+ /**
		499	+ * Returns a reference to the main class
		500	+ *
		501	+ * @return Indexer
		502	+ */
		503	+ public static function get()
		504	+ {
		505	+ static $singleton = null;
		506	+
		507	+ if (is_null($singleton))
		508	+ {
		509	+ $config = KTConfig::getSingleton();
		510	+ $classname = $config->get('indexer/coreClass');
		511	+
		512	+ require_once('indexing/indexers/' . $classname . '.inc.php');
		513	+
		514	+ if (!class_exists($classname))
		515	+ {
		516	+ throw new Exception("Class '$classname' does not exist.");
		517	+ }
		518	+
		519	+ $singleton = new $classname;
		520	+ }
		521	+
		522	+ return $singleton;
		523	+ }
		524	+
		525	+ public abstract function deleteDocument($docid);
		526	+
		527	+ /**
		528	+ * Remove the association of all extractors to mime types on the database.
		529	+ *
		530	+ */
		531	+ public function clearExtractors()
		532	+ {
		533	+ global $default;
		534	+
		535	+ $sql = "update mime_types set extractor_id=null";
		536	+ DBUtil::runQuery($sql);
		537	+
		538	+ $sql = "delete from mime_extractors";
		539	+ DBUtil::runQuery($sql);
		540	+
		541	+ if ($this->debug) $default->log->debug('clearExtractors');
		542	+ }
		543	+
		544	+ /**
		545	+ * lookup the name of the extractor class based on the mime type.
		546	+ *
		547	+ * @param string $type
		548	+ * @return string
		549	+ */
		550	+ public static function resolveExtractor($type)
		551	+ {
		552	+ global $default;
		553	+ $sql = "select extractor from mime_types where filetypes='$type'";
		554	+ $class = DBUtil::getOneResultKey($sql,'extractor');
		555	+ if (PEAR::isError($class))
		556	+ {
		557	+ $default->log->error("resolveExtractor: cannot resolve $type");
		558	+ return $class;
		559	+ }
		560	+ if ($this->debug) $default->log->debug(sprintf(_kt("resolveExtractor: Resolved '%s' from mime type '%s'."), $class, $type));
		561	+ return $class;
		562	+ }
		563	+
		564	+ /**
		565	+ * Return all the discussion text.
		566	+ *
		567	+ * @param int $docid
		568	+ * @return string
		569	+ */
		570	+ public static function getDiscussionText($docid)
		571	+ {
		572	+ $sql = "SELECT
		573	+ dc.subject, dc.body
		574	+ FROM
		575	+ discussion_threads dt
		576	+ INNER JOIN discussion_comments dc ON dc.thread_id=dt.id AND dc.id BETWEEN dt.first_comment_id AND dt.last_comment_id
		577	+ WHERE
		578	+ dt.document_id=$docid";
		579	+ $result = DBUtil::getResultArray($sql);
		580	+ $text = '';
		581	+
		582	+ foreach($result as $record)
		583	+ {
		584	+ $text .= $record['subject'] . "\n" . $record['body'] . "\n";
		585	+ }
		586	+
		587	+ return $text;
		588	+ }
		589	+
		590	+ /**
		591	+ * Schedule the indexing of a document.
		592	+ *
		593	+ * @param string $document
		594	+ * @param string $what
		595	+ */
		596	+ public static function index($document, $what='A')
		597	+ {
		598	+ global $default;
		599	+
		600	+ if (is_numeric($document))
		601	+ {
		602	+ $document = Document::get($document+0);
		603	+ }
		604	+
		605	+ if (PEAR::isError($document))
		606	+ {
		607	+ $default->log->error("index: Could not index document: " .$document->getMessage());
		608	+ return;
		609	+ }
		610	+
		611	+ $document_id = $document->getId();
		612	+ $userid=$_SESSION['userID'];
		613	+ if (empty($userid)) $userid=1;
		614	+
		615	+ // we dequeue the document so that there are no issues when enqueuing
		616	+ Indexer::unqueueDocument($document_id);
		617	+
		618	+ // enqueue item
		619	+ $sql = "INSERT INTO index_files(document_id, user_id, what) VALUES($document_id, $userid, '$what')";
		620	+ DBUtil::runQuery($sql);
		621	+
		622	+ $default->log->debug("index: Queuing indexing of $document_id");
		623	+
		624	+ }
		625	+
		626	+ private static function incrementCount()
		627	+ {
		628	+ // Get count from system settings
		629	+ $count = Indexer::getIndexedDocumentCount();
		630	+ $count = (int)$count + 1;
		631	+ Indexer::updateIndexedDocumentCount($count);
		632	+ }
		633	+
		634	+ public static function getIndexedDocumentCount()
		635	+ {
		636	+ $count = KTUtil::getSystemSetting('indexedDocumentCount', 0);
		637	+ return (int) $count;
		638	+ }
		639	+
		640	+ public static function updateIndexedDocumentCount($cnt = 0)
		641	+ {
		642	+ KTUtil::setSystemSetting('indexedDocumentCount', $cnt);
		643	+ }
		644	+
		645	+ public static function reindexQueue()
		646	+ {
		647	+ $sql = "UPDATE index_files SET processdate = null";
		648	+ DBUtil::runQuery($sql);
		649	+ }
		650	+
		651	+ public static function reindexDocument($documentId)
		652	+ {
		653	+ $sql = "UPDATE index_files SET processdate=null, status_msg=null WHERE document_id=$documentId";
		654	+ DBUtil::runQuery($sql);
		655	+ }
		656	+
		657	+
		658	+
		659	+ public static function indexAll()
		660	+ {
		661	+ $userid=$_SESSION['userID'];
		662	+ if (empty($userid)) $userid=1;
		663	+
		664	+ $sql = "DELETE FROM index_files";
		665	+ DBUtil::runQuery($sql);
		666	+
		667	+ $sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'A' FROM documents WHERE status_id=1 and id not in (select document_id from index_files)";
		668	+ DBUtil::runQuery($sql);
		669	+ }
		670	+
		671	+ public static function indexFolder($folder)
		672	+ {
		673	+ $userid=$_SESSION['userID'];
		674	+ if (empty($userid)) $userid=1;
		675	+
		676	+ if (!$folder instanceof Folder && !$folder instanceof FolderProxy)
		677	+ {
		678	+ throw new Exception('Folder expected');
		679	+ }
		680	+
		681	+ $full_path = $folder->getFullPath();
		682	+
		683	+ $sql = "INSERT INTO index_files(document_id, user_id, what) SELECT id, $userid, 'A' FROM documents WHERE full_path like '{$full_path}/%' AND status_id=1 and id not in (select document_id from index_files)";
		684	+ DBUtil::runQuery($sql);
		685	+ }
		686	+
		687	+ /**
		688	+ * Clearout the scheduling of documents that no longer exist.
		689	+ *
		690	+ */
		691	+ public static function clearoutDeleted()
		692	+ {
		693	+ global $default;
		694	+
		695	+ $sql = 'DELETE FROM
		696	+ index_files
		697	+ WHERE
		698	+ document_id in (SELECT d.id FROM documents AS d WHERE d.status_id=3) OR
		699	+ NOT EXISTS(SELECT index_files.document_id FROM documents WHERE index_files.document_id=documents.id)';
		700	+ DBUtil::runQuery($sql);
		701	+
		702	+ $default->log->debug("Indexer::clearoutDeleted: removed documents from indexing queue that have been deleted");
		703	+ }
		704	+
		705	+
		706	+ /**
		707	+ * Check if a document is scheduled to be indexed
		708	+ *
		709	+ * @param mixed $document This may be a document or document id
		710	+ * @return boolean
		711	+ */
		712	+ public static function isDocumentScheduled($document)
		713	+ {
		714	+ if (is_numeric($document))
		715	+ {
		716	+ $docid = $document;
		717	+ }
		718	+ else if ($document instanceof Document)
		719	+ {
		720	+ $docid = $document->getId();
		721	+ }
		722	+ else
		723	+ {
		724	+ return false;
		725	+ }
		726	+ $sql = "SELECT 1 FROM index_files WHERE document_id=$docid";
		727	+ $result = DBUtil::getResultArray($sql);
		728	+ return count($result) > 0;
		729	+ }
		730	+
		731	+ /**
		732	+ * Filters text removing redundant characters such as continuous newlines and spaces.
		733	+ *
		734	+ * @param string $filename
		735	+ */
		736	+ private function filterText($filename)
		737	+ {
		738	+ $content = file_get_contents($filename);
		739	+
		740	+ $src = array("([\r\n])","([\n][\n])","([\n])","([\t])",'([ ][ ])');
		741	+ $tgt = array("\n","\n",' ',' ',' ');
		742	+
		743	+ // shrink what is being stored.
		744	+ do
		745	+ {
		746	+ $orig = $content;
		747	+ $content = preg_replace($src, $tgt, $content);
		748	+ } while ($content != $orig);
		749	+
		750	+ return file_put_contents($filename, $content) !== false;
		751	+ }
		752	+
		753	+ /**
		754	+ * Load hooks for text extraction process.
		755	+ *
		756	+ */
		757	+ private function loadExtractorHooks()
		758	+ {
		759	+ $this->generalHookCache = array();
		760	+ $this->mimeHookCache = array();
		761	+
		762	+
		763	+ $dir = opendir(SearchHelper::correctPath($this->hookPath));
		764	+ while (($file = readdir($dir)) !== false)
		765	+ {
		766	+ if (substr($file,-12) == 'Hook.inc.php')
		767	+ {
		768	+ require_once($this->hookPath . '/' . $file);
		769	+ $class = substr($file, 0, -8);
		770	+
		771	+ if (!class_exists($class))
		772	+ {
		773	+ continue;
		774	+ }
		775	+
		776	+ $hook = new $class;
		777	+ if (!($class instanceof ExtractorHook))
		778	+ {
		779	+ continue;
		780	+ }
		781	+
		782	+ $mimeTypes = $hook->registerMimeTypes();
		783	+ if (is_null($mimeTypes))
		784	+ {
		785	+ $this->generalHookCache[] = & $hook;
		786	+ }
		787	+ else
		788	+ {
		789	+ foreach($mimeTypes as $type)
		790	+ {
		791	+ $this->mimeHookCache[$type][] = & $hook;
		792	+ }
		793	+ }
		794	+
		795	+ }
		796	+ }
		797	+ closedir($dir);
		798	+ }
		799	+
		800	+ /**
		801	+ * This is a refactored function to execute the hooks.
		802	+ *
		803	+ * @param DocumentExtractor $extractor
		804	+ * @param string $phase
		805	+ * @param string $mimeType Optional. If set, indicates which hooks must be used, else assume general.
		806	+ */
		807	+ private function executeHook($extractor, $phase, $mimeType = null)
		808	+ {
		809	+ $hooks = array();
		810	+ if (is_null($mimeType))
		811	+ {
		812	+ $hooks = $this->generalHookCache;
		813	+ }
		814	+ else
		815	+ {
		816	+ if (array_key_exists($mimeType, $this->mimeHookCache))
		817	+ {
		818	+ $hooks = $this->mimeHookCache[$mimeType];
		819	+ }
		820	+ }
		821	+ if (empty($hooks))
		822	+ {
		823	+ return;
		824	+ }
		825	+
		826	+ foreach($hooks as $hook)
		827	+ {
		828	+ $hook->$phase($extractor);
		829	+ }
		830	+ }
		831	+
		832	+ private function doesDiagnosticsPass($simple=false)
		833	+ {
		834	+ global $default;
		835	+
		836	+ $config =& KTConfig::getSingleton();
		837	+ // create a index log lock file in case there are errors, and we don't need to log them forever!
		838	+ // this function will create the lockfile if an error is detected. It will be removed as soon
		839	+ // as the problems with the indexer are removed.
		840	+ $lockFile = $config->get('cache/cacheDirectory') . '/index.log.lock';
		841	+
		842	+ $diagnosis = $this->diagnose();
		843	+ if (!is_null($diagnosis))
		844	+ {
		845	+ if (!is_file($lockFile))
		846	+ {
		847	+ $default->log->error(_kt('Indexer problem: ') . $diagnosis);
		848	+ }
		849	+ touch($lockFile);
		850	+ return false;
		851	+ }
		852	+
		853	+ if ($simple)
		854	+ {
		855	+ return true;
		856	+ }
		857	+
		858	+ $diagnosis = $this->diagnoseExtractors();
		859	+ if (!empty($diagnosis))
		860	+ {
		861	+ if (!is_file($lockFile))
		862	+ {
		863	+ foreach($diagnosis as $diag)
		864	+ {
		865	+ $default->log->error(sprintf(_kt('%s problem: %s'), $diag['name'],$diag['diagnosis']));
		866	+ }
		867	+ }
		868	+ touch($lockFile);
		869	+ return false;
		870	+ }
		871	+
		872	+ if (is_file($lockFile))
		873	+ {
		874	+ $default->log->info(_kt('Issues with the indexer have been resolved!'));
		875	+ unlink($lockFile);
		876	+ }
		877	+
		878	+ return true;
		879	+ }
		880	+
		881	+ /**
		882	+ * This does the initial mime type association between mime types and text extractors
		883	+ *
		884	+ */
		885	+ public function checkForRegisteredTypes()
		886	+ {
		887	+ global $default;
		888	+
		889	+ // we are only doing this once!
		890	+ $initRegistered = KTUtil::getSystemSetting('mimeTypesRegistered', false);
		891	+ if ($initRegistered)
		892	+ {
		893	+ return;
		894	+ }
		895	+ if ($this->debug) $default->log->debug('checkForRegisteredTypes: start');
		896	+
		897	+ $date = date('Y-m-d H:i');
		898	+ $sql = "UPDATE scheduler_tasks SET run_time='$date'";
		899	+ DBUtil::runQuery($sql);
		900	+
		901	+ $this->registerTypes(true);
		902	+
		903	+ $disable = array(
		904	+ 'windows'=>array('PSExtractor'),
		905	+ 'unix' => array()
		906	+ );
		907	+
		908	+ $disableForOS = OS_WINDOWS?$disable['windows']:$disable['unix'];
		909	+
		910	+ if (!empty($disableForOS))
		911	+ {
		912	+ $disableForOS = '\'' . implode("','", $disableForOS) .'\'';
		913	+
		914	+ $sql = "UPDATE mime_extractors SET active=0 WHERE name in ($disableForOS)";
		915	+ DBUtil::runQuery($sql);
		916	+ $default->log->info("checkForRegisteredTypes: disabled '$extractor'");
		917	+ }
		918	+ $this->loadExtractorStatus();
		919	+
		920	+ if ($this->debug) $default->log->debug('checkForRegisteredTypes: done');
		921	+ KTUtil::setSystemSetting('mimeTypesRegistered', true);
		922	+ }
		923	+
		924	+ private function updatePendingDocumentStatus($documentId, $message, $level)
		925	+ {
		926	+ $this->indexingHistory .= "\n" . $level . ': ' . $message;
		927	+ $message = sanitizeForSQL($this->indexingHistory);
		928	+ $sql = "UPDATE index_files SET status_msg='$message' WHERE document_id=$documentId";
		929	+ DBUtil::runQuery($sql);
		930	+ }
		931	+
		932	+ private $restartCurrentBatch = false;
		933	+
		934	+ public function restartBatch()
		935	+ {
		936	+ $this->restartCurrentBatch = true;
		937	+ }
		938	+
		939	+ /**
		940	+ *
		941	+ * @param int $documentId
		942	+ * @param string $message
		943	+ * @param string $level This may be info, error, debug
		944	+ */
		945	+ private function logPendingDocumentInfoStatus($documentId, $message, $level)
		946	+ {
		947	+ $this->updatePendingDocumentStatus($documentId, $message, $level);
		948	+ global $default;
		949	+
		950	+ switch ($level)
		951	+ {
		952	+ case 'debug':
		953	+ if ($this->debug)
		954	+ {
		955	+ $default->log->debug($message);
		956	+ }
		957	+ break;
		958	+ default:
		959	+ $default->log->$level($message);
		960	+ }
		961	+ }
		962	+
		963	+
		964	+
		965	+ public function getExtractor($extractorClass)
		966	+ {
		967	+ if (empty($extractorClass))
		968	+ {
		969	+ return null;
		970	+ }
		971	+
		972	+ $includeFile = SEARCH2_INDEXER_DIR . 'extractors/' . $extractorClass . '.inc.php';
		973	+ if (!file_exists($includeFile))
		974	+ {
		975	+ throw new Exception("Extractor file does not exist: $includeFile");
		976	+ }
		977	+
		978	+ require_once($includeFile);
		979	+
		980	+ if (!class_exists($extractorClass))
		981	+ {
		982	+ throw new Exception("Extractor '$classname' not defined in file: $includeFile");
		983	+ }
		984	+
		985	+ $extractor = new $extractorClass();
		986	+
		987	+ if (!($extractor instanceof DocumentExtractor))
		988	+ {
		989	+ throw new Exception("Class $classname was expected to be of type DocumentExtractor");
		990	+ }
		991	+
		992	+ return $extractor;
		993	+ }
		994	+
		995	+ public static function getIndexingQueue($problemItemsOnly=true)
		996	+ {
		997	+
		998	+ if ($problemItemsOnly)
		999	+ {
		1000	+ $sql = "SELECT
		1001	+ iff.document_id, iff.indexdate, mt.filetypes, mt.mimetypes, me.name as extractor, iff.what, iff.status_msg, dcv.filename
		1002	+ FROM
		1003	+ index_files iff
		1004	+ INNER JOIN documents d ON iff.document_id=d.id
		1005	+ INNER JOIN document_metadata_version dmv ON d.metadata_version_id=dmv.id
		1006	+ INNER JOIN document_content_version dcv ON dmv.content_version_id=dcv.id
		1007	+ INNER JOIN mime_types mt ON dcv.mime_id=mt.id
		1008	+ LEFT JOIN mime_extractors me ON mt.extractor_id=me.id
		1009	+ WHERE
		1010	+ (iff.status_msg IS NOT NULL AND iff.status_msg <> '') AND d.status_id=1
		1011	+ ORDER BY indexdate ";
		1012	+ }
		1013	+ else
		1014	+ {
		1015	+ $sql = "SELECT
		1016	+ iff.document_id, iff.indexdate, mt.filetypes, mt.mimetypes, me.name as extractor, iff.what, iff.status_msg, dcv.filename
		1017	+ FROM
		1018	+ index_files iff
		1019	+ INNER JOIN documents d ON iff.document_id=d.id
		1020	+ INNER JOIN document_metadata_version dmv ON d.metadata_version_id=dmv.id
		1021	+ INNER JOIN document_content_version dcv ON dmv.content_version_id=dcv.id
		1022	+ INNER JOIN mime_types mt ON dcv.mime_id=mt.id
		1023	+ LEFT JOIN mime_extractors me ON mt.extractor_id=me.id
		1024	+ WHERE
		1025	+ (iff.status_msg IS NULL or iff.status_msg = '') AND d.status_id=1
		1026	+ ORDER BY indexdate ";
		1027	+ }
		1028	+ $aResult = DBUtil::getResultArray($sql);
		1029	+
		1030	+ return $aResult;
		1031	+ }
		1032	+
		1033	+ public static function getPendingIndexingQueue()
		1034	+ {
		1035	+ return Indexer::getIndexingQueue(false);
		1036	+ }
		1037	+
		1038	+ public function updateIndexStats()
		1039	+ {
		1040	+ $optimisationDate = KTUtil::getSystemSetting('luceneOptimisationDate', '');
		1041	+
		1042	+ $noOptimisation = false;
		1043	+ if ($optimisationDate == '')
		1044	+ {
		1045	+ $optimisationDate = _kt('N/A');
		1046	+ $optimisationPeriod = $optimisationDate;
		1047	+ }
		1048	+ else
		1049	+ {
		1050	+ $optimisationPeriod = KTUtil::computePeriodToDate($optimisationDate, null, true);
		1051	+ $noOptimisation = $optimisationPeriod['days'] > 2;
		1052	+ $optimisationPeriod = $optimisationPeriod['str'];
		1053	+ $optimisationDate = date('Y-m-d H:i:s', $optimisationDate);
		1054	+ }
		1055	+
		1056	+ $indexingDate = KTUtil::getSystemSetting('luceneIndexingDate', '');
		1057	+ if ($indexingDate == '')
		1058	+ {
		1059	+ $indexingDate = _kt('N/A');
		1060	+ $indexingPeriod = $indexingDate;
		1061	+ }
		1062	+ else
		1063	+ {
		1064	+ $indexingPeriod = KTUtil::computePeriodToDate($indexingDate);
		1065	+ $indexingDate = date('Y-m-d H:i:s', $indexingDate);
		1066	+ }
		1067	+
		1068	+ $index = Indexer::get();
		1069	+ $docsInIndex = $index->getDocumentsInIndex();
		1070	+
		1071	+ // we are only interested in documents that are active
		1072	+ $sql = "SELECT count(*) as docsInQueue FROM index_files i inner join documents d on i.document_id = d.id where (i.status_msg is null or i.status_msg = '') and d.status_id=1";
		1073	+ $docsInQueue = DBUtil::getOneResultKey($sql, 'docsInQueue');
		1074	+
		1075	+ $sql = "SELECT count(*) as errorsInQueue FROM index_files i inner join documents d on i.document_id = d.id where (i.status_msg is not null or i.status_msg <> '') and d.status_id=1";
		1076	+ $errorsInQueue = DBUtil::getOneResultKey($sql, 'errorsInQueue');
		1077	+
		1078	+ $sql = "SELECT count(*) as docsInRepository FROM documents where status_id=1";
		1079	+ $docsInRepository = DBUtil::getOneResultKey($sql, 'docsInRepository');
		1080	+
		1081	+ if ($docsInRepository == 0)
		1082	+ {
		1083	+ $indexingCoverage = '0.00%';
		1084	+ $queueCoverage = $indexingCoverage;
		1085	+ }
		1086	+ else
		1087	+ {
		1088	+ // compute indexing coverage
		1089	+ $indexingCoverage = _kt('Not Available');
		1090	+ if (is_numeric($docsInIndex))
		1091	+ {
		1092	+ $indexingCoverage = ($docsInIndex * 100) / $docsInRepository;
		1093	+ $indexingCoverage = number_format($indexingCoverage, 2, '.',',') . '%';
		1094	+ }
		1095	+
		1096	+ // compute queue coverage
		1097	+ $queueCoverage = _kt('Not Available');
		1098	+ if (is_numeric($docsInQueue))
		1099	+ {
		1100	+ $queueCoverage = ($docsInQueue * 100) / $docsInRepository;
		1101	+ $queueCoverage = number_format($queueCoverage, 2, '.',',') . '%';
		1102	+ }
		1103	+ }
		1104	+
		1105	+
		1106	+ $stats = array(
		1107	+ 'optimisationDate'=>$optimisationDate,
		1108	+ 'optimisationPeriod'=>$optimisationPeriod,
		1109	+ 'indexingDate'=>$indexingDate,
		1110	+ 'indexingPeriod'=>$indexingPeriod,
		1111	+ 'docsInIndex'=>$docsInIndex,
		1112	+ 'docsInQueue'=>$docsInQueue,
		1113	+ 'errorsInQueue'=>$errorsInQueue,
		1114	+ 'docsInRepository'=>$docsInRepository,
		1115	+ 'indexingCoverage'=>$indexingCoverage,
		1116	+ 'queueCoverage'=>$queueCoverage,
		1117	+ 'noOptimisation'=>$noOptimisation
		1118	+ );
		1119	+
		1120	+ KTUtil::setSystemSetting('indexerStats', serialize($stats));
		1121	+
		1122	+ $indexer = Indexer::get();
		1123	+
		1124	+ $diagnosis = $indexer->diagnose();
		1125	+ KTUtil::setSystemSetting('indexerDiagnostics', serialize($diagnosis));
		1126	+
		1127	+ $extractorDiagnosis = $indexer->diagnoseExtractors();
		1128	+
		1129	+ KTUtil::setSystemSetting('extractorDiagnostics', serialize($extractorDiagnosis));
		1130	+ }
		1131	+
		1132	+ /**
		1133	+ * The main function that may be called repeatedly to index documents.
		1134	+ *
		1135	+ * @param int $max Default 20
		1136	+ */
		1137	+ public function indexDocuments($max=null)
		1138	+ {
		1139	+ global $default;
		1140	+ $config =& KTConfig::getSingleton();
		1141	+
		1142	+ /*$indexLockFile = $config->get('cache/cacheDirectory') . '/main.index.lock';
		1143	+ if (is_file($indexLockFile))
		1144	+ {
		1145	+ $default->log->info('indexDocuments: main.index.lock seems to exist. it could be that the indexing is still underway.');
		1146	+ $default->log->info('indexDocuments: Remove "' . $indexLockFile . '" if the indexing is not running or extend the frequency at which the background task runs!');
		1147	+ return;
		1148	+ }
		1149	+ touch($indexLockFile);*/
		1150	+
		1151	+
		1152	+ $this->checkForRegisteredTypes();
		1153	+
		1154	+ if ($this->debug) $default->log->debug('indexDocuments: start');
		1155	+ if (!$this->doesDiagnosticsPass())
		1156	+ {
		1157	+ //unlink($indexLockFile);
		1158	+ if ($this->debug) $default->log->debug('indexDocuments: stopping - diagnostics problem. The dashboard will provide more information.');
		1159	+ return;
		1160	+ }
		1161	+
		1162	+ if (is_null($max))
		1163	+ {
		1164	+ $max = $config->get('indexer/batchDocuments',20);
		1165	+ }
		1166	+
		1167	+ $this->loadExtractorHooks();
		1168	+
		1169	+ Indexer::clearoutDeleted();
		1170	+
		1171	+ $date = date('Y-m-d H:i:s');
		1172	+ // identify the indexers that must run
		1173	+ // mysql specific limit!
		1174	+ $sql = "SELECT
		1175	+ iff.document_id, mt.filetypes, mt.mimetypes, me.name as extractor, iff.what
		1176	+ FROM
		1177	+ index_files iff
		1178	+ INNER JOIN documents d ON iff.document_id=d.id
		1179	+ INNER JOIN document_metadata_version dmv ON d.metadata_version_id=dmv.id
		1180	+ INNER JOIN document_content_version dcv ON dmv.content_version_id=dcv.id
		1181	+ INNER JOIN mime_types mt ON dcv.mime_id=mt.id
		1182	+ LEFT JOIN mime_extractors me ON mt.extractor_id=me.id
		1183	+ WHERE
		1184	+ (iff.processdate IS NULL or iff.processdate < date_sub('$date', interval 1 day)) AND dmv.status_id=1
		1185	+ ORDER BY indexdate
		1186	+ LIMIT $max";
		1187	+ $result = DBUtil::getResultArray($sql);
		1188	+ if (PEAR::isError($result))
		1189	+ {
		1190	+ //unlink($indexLockFile);
		1191	+ if ($this->debug) $default->log->debug('indexDocuments: stopping - db error');
		1192	+ return;
		1193	+ }
		1194	+ KTUtil::setSystemSetting('luceneIndexingDate', time());
		1195	+
		1196	+ // bail if no work to do
		1197	+ if (count($result) == 0)
		1198	+ {
		1199	+ //unlink($indexLockFile);
		1200	+ if ($this->debug) $default->log->debug('indexDocuments: stopping - no work to be done');
		1201	+ return;
		1202	+ }
		1203	+
		1204	+ // identify any documents that need indexing and mark them
		1205	+ // so they are not taken in a followup run
		1206	+ $ids = array();
		1207	+ foreach($result as $docinfo)
		1208	+ {
		1209	+ $ids[] = $docinfo['document_id'];
		1210	+ }
		1211	+
		1212	+ // mark the documents as being processed
		1213	+
		1214	+ $ids=implode(',',$ids);
		1215	+ $sql = "UPDATE index_files SET processdate='$date' WHERE document_id in ($ids)";
		1216	+ DBUtil::runQuery($sql);
		1217	+
		1218	+ $extractorCache = array();
		1219	+ $storageManager = KTStorageManagerUtil::getSingleton();
		1220	+
		1221	+ $tempPath = $config->get("urls/tmpDirectory");
		1222	+
		1223	+ foreach($result as $docinfo)
		1224	+ {
		1225	+ // increment indexed documents count
		1226	+ Indexer::incrementCount();
		1227	+
		1228	+ $docId=$docinfo['document_id'];
		1229	+ $extension=$docinfo['filetypes'];
		1230	+ $mimeType=$docinfo['mimetypes'];
		1231	+ $extractorClass=$docinfo['extractor'];
		1232	+ $indexDocument = in_array($docinfo['what'], array('A','C'));
		1233	+ $indexDiscussion = in_array($docinfo['what'], array('A','D'));
		1234	+ $this->indexingHistory = '';
		1235	+
		1236	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Indexing docid: %d extension: '%s' mimetype: '%s' extractor: '%s'"), $docId, $extension,$mimeType,$extractorClass), 'debug');
		1237	+
		1238	+ if (empty($extractorClass))
		1239	+ {
		1240	+ /*
		1241	+
		1242	+ if no extractor is found and we don't need to index discussions, then we can remove the item from the queue.
		1243	+
		1244	+ */
		1245	+ if ($indexDiscussion)
		1246	+ {
		1247	+ $indexDocument = false;
		1248	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Not indexing docid: %d content because extractor could not be resolve. Still indexing discussion."), $docId), 'info');
		1249	+ }
		1250	+ else
		1251	+ {
		1252	+ Indexer::unqueueDocument($docId, sprintf(_kt("No extractor for docid: %d"),$docId));
		1253	+ continue;
		1254	+ }
		1255	+ }
		1256	+ else
		1257	+ {
		1258	+ /*
		1259	+
		1260	+ If an extractor is available, we must ensure it is enabled.
		1261	+
		1262	+ */
		1263	+
		1264	+ if (!$this->isExtractorEnabled($extractorClass))
		1265	+ {
		1266	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("diagnose: Not indexing docid: %d because extractor '%s' is disabled."), $docId, $extractorClass), 'info');
		1267	+ continue;
		1268	+ }
		1269	+ }
		1270	+
		1271	+ if ($this->debug)
		1272	+ {
		1273	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Processing docid: %d.\n"),$docId), 'info');
		1274	+ }
		1275	+
		1276	+ $document = Document::get($docId);
		1277	+ if (PEAR::isError($document))
		1278	+ {
		1279	+ Indexer::unqueueDocument($docId,sprintf(_kt("indexDocuments: Cannot resolve document id %d: %s."),$docId, $document->getMessage()), 'error');
		1280	+ continue;
		1281	+ }
		1282	+
		1283	+ if ($this->restartCurrentBatch)
		1284	+ {
		1285	+ Indexer::unqueueDocument($docId);
		1286	+ Indexer::index($docId, 'A');
		1287	+ continue;
		1288	+ }
		1289	+
		1290	+
		1291	+ $filename = $document->getFileName();
		1292	+ if (substr($filename,0,1) == '~' \|\| substr($filename,-1) == '~')
		1293	+ {
		1294	+ Indexer::unqueueDocument($docId,sprintf(_kt("indexDocuments: Filename for document id %d starts with a tilde (~). This is assumed to be a temporary file. This is ignored."),$docId), 'error');
		1295	+ continue;
		1296	+ }
		1297	+
		1298	+ $removeFromQueue = true;
		1299	+ if ($indexDocument)
		1300	+ {
		1301	+ if (array_key_exists($extractorClass, $extractorCache))
		1302	+ {
		1303	+ $extractor = $extractorCache[$extractorClass];
		1304	+ }
		1305	+ else
		1306	+ {
		1307	+ $extractor = $extractorCache[$extractorClass] = $this->getExtractor($extractorClass);
		1308	+ }
		1309	+
		1310	+ if (!($extractor instanceof DocumentExtractor))
		1311	+ {
		1312	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("indexDocuments: extractor '%s' is not a document extractor class."),$extractorClass), 'error');
		1313	+ continue;
		1314	+ }
		1315	+
		1316	+
		1317	+
		1318	+ $version = $document->getMajorVersionNumber() . '.' . $document->getMinorVersionNumber();
		1319	+ $sourceFile = $storageManager->temporaryFile($document);
		1320	+
		1321	+ if (empty($sourceFile) \|\| !is_file($sourceFile))
		1322	+ {
		1323	+ Indexer::unqueueDocument($docId,sprintf(_kt("indexDocuments: source file '%s' for document %d does not exist."),$sourceFile,$docId), 'error');
		1324	+ continue;
		1325	+ }
		1326	+
		1327	+ if ($extractor->needsIntermediateSourceFile())
		1328	+ {
		1329	+ //$extension = pathinfo($document->getFileName(), PATHINFO_EXTENSION);
		1330	+
		1331	+ $intermediate = $tempPath . '/'. $docId . '.' . $extension;
		1332	+ $result = @copy($sourceFile, $intermediate);
		1333	+ if ($result === false)
		1334	+ {
		1335	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Could not create intermediate file from document %d"),$docId), 'error');
		1336	+ // problem. lets try again later. probably permission related. log the issue.
		1337	+ continue;
		1338	+ }
		1339	+ $sourceFile = $intermediate;
		1340	+ }
		1341	+
		1342	+ $targetFile = tempnam($tempPath, 'ktindexer');
		1343	+
		1344	+ $extractor->setSourceFile($sourceFile);
		1345	+ $extractor->setMimeType($mimeType);
		1346	+ $extractor->setExtension($extension);
		1347	+ $extractor->setTargetFile($targetFile);
		1348	+ $extractor->setDocument($document);
		1349	+ $extractor->setIndexingStatus(null);
		1350	+ $extractor->setExtractionStatus(null);
		1351	+
		1352	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Extra Info docid: %d Source File: '%s' Target File: '%s'"),$docId,$sourceFile,$targetFile), 'debug');
		1353	+
		1354	+ $this->executeHook($extractor, 'pre_extract');
		1355	+ $this->executeHook($extractor, 'pre_extract', $mimeType);
		1356	+ $removeFromQueue = false;
		1357	+
		1358	+ if ($extractor->extractTextContent())
		1359	+ {
		1360	+ // the extractor may need to create another target file
		1361	+ $targetFile = $extractor->getTargetFile();
		1362	+
		1363	+ $extractor->setExtractionStatus(true);
		1364	+ $this->executeHook($extractor, 'pre_index');
		1365	+ $this->executeHook($extractor, 'pre_index', $mimeType);
		1366	+
		1367	+ $title = $document->getName();
		1368	+ if ($indexDiscussion)
		1369	+ {
		1370	+ if (!$this->filterText($targetFile))
		1371	+ {
		1372	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"),$docId), 'error');
		1373	+ }
		1374	+ else
		1375	+ {
		1376	+ $indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version);
		1377	+ $removeFromQueue = $indexStatus;
		1378	+ if (!$indexStatus)
		1379	+ {
		1380	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocumentAndDiscussion"),$docId), 'error');
		1381	+ }
		1382	+
		1383	+ $extractor->setIndexingStatus($indexStatus);
		1384	+ }
		1385	+ }
		1386	+ else
		1387	+ {
		1388	+ if (!$this->filterText($targetFile))
		1389	+ {
		1390	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem filtering document %d"),$docId), 'error');
		1391	+ }
		1392	+ else
		1393	+ {
		1394	+ $indexStatus = $this->indexDocument($docId, $targetFile, $title, $version);
		1395	+ $removeFromQueue = $indexStatus;
		1396	+
		1397	+ if (!$indexStatus)
		1398	+ {
		1399	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Problem indexing document %d - indexDocument"),$docId), 'error');
		1400	+ $this->logPendingDocumentInfoStatus($docId, '<output>' . $extractor->output . '</output>', 'error');
		1401	+ }
		1402	+
		1403	+ $extractor->setIndexingStatus($indexStatus);
		1404	+ }
		1405	+ }
		1406	+
		1407	+ $this->executeHook($extractor, 'post_index', $mimeType);
		1408	+ $this->executeHook($extractor, 'post_index');
		1409	+ }
		1410	+ else
		1411	+ {
		1412	+ $extractor->setExtractionStatus(false);
		1413	+ $this->logPendingDocumentInfoStatus($docId, sprintf(_kt("Could not extract contents from document %d"),$docId), 'error');
		1414	+ $this->logPendingDocumentInfoStatus($docId, '<output>' . $extractor->output . '</output>', 'error');
		1415	+ }
		1416	+
		1417	+ $this->executeHook($extractor, 'post_extract', $mimeType);
		1418	+ $this->executeHook($extractor, 'post_extract');
		1419	+
		1420	+ if ($extractor->needsIntermediateSourceFile())
		1421	+ {
		1422	+ @unlink($sourceFile);
		1423	+ }
		1424	+
		1425	+ @unlink($targetFile);
		1426	+
		1427	+ }
		1428	+ else
		1429	+ {
		1430	+ $indexStatus = $this->indexDiscussion($docId);
		1431	+ $removeFromQueue = $indexStatus;
		1432	+ }
		1433	+
		1434	+ if ($removeFromQueue)
		1435	+ {
		1436	+ Indexer::unqueueDocument($docId, sprintf(_kt("Done indexing docid: %d"),$docId));
		1437	+ }
		1438	+ else
		1439	+ {
		1440	+ if ($this->debug) $default->log->debug(sprintf(_kt("Document docid: %d was not removed from the queue as it looks like there was a problem with the extraction process"),$docId));
		1441	+ }
		1442	+ }
		1443	+ if ($this->debug) $default->log->debug('indexDocuments: done');
		1444	+ //unlink($indexLockFile);
		1445	+ }
		1446	+
		1447	+ public function migrateDocuments($max=null)
		1448	+ {
		1449	+ global $default;
		1450	+
		1451	+ $default->log->info(_kt('migrateDocuments: starting'));
		1452	+
		1453	+ if (!$this->doesDiagnosticsPass(true))
		1454	+ {
		1455	+ $default->log->info(_kt('migrateDocuments: stopping - diagnostics problem. The dashboard will provide more information.'));
		1456	+ return;
		1457	+ }
		1458	+
		1459	+ if (KTUtil::getSystemSetting('migrationComplete') == 'true')
		1460	+ {
		1461	+ $default->log->info(_kt('migrateDocuments: stopping - migration is complete.'));
		1462	+ return;
		1463	+ }
		1464	+
		1465	+ $config =& KTConfig::getSingleton();
		1466	+ if (is_null($max))
		1467	+ {
		1468	+ $max = $config->get('indexer/batchMigrateDocument',500);
		1469	+ }
		1470	+
		1471	+ $lockFile = $config->get('cache/cacheDirectory') . '/migration.lock';
		1472	+ if (is_file($lockFile))
		1473	+ {
		1474	+ $default->log->info(_kt('migrateDocuments: stopping - migration lockfile detected.'));
		1475	+ return;
		1476	+ }
		1477	+ touch($lockFile);
		1478	+
		1479	+ $startTime = KTUtil::getSystemSetting('migrationStarted');
		1480	+ if (is_null($startTime))
		1481	+ {
		1482	+ KTUtil::setSystemSetting('migrationStarted', time());
		1483	+ }
		1484	+
		1485	+ $maxLoops = 5;
		1486	+
		1487	+ $max = ceil($max / $maxLoops);
		1488	+
		1489	+ $start =KTUtil::getBenchmarkTime();
		1490	+ $noDocs = false;
		1491	+ $numDocs = 0;
		1492	+
		1493	+ for($loop=0;$loop<$maxLoops;$loop++)
		1494	+ {
		1495	+
		1496	+ $sql = "SELECT
		1497	+ document_id, document_text
		1498	+ FROM
		1499	+ document_text
		1500	+ ORDER BY document_id
		1501	+ LIMIT $max";
		1502	+ $result = DBUtil::getResultArray($sql);
		1503	+ if (PEAR::isError($result))
		1504	+ {
		1505	+ $default->log->info(_kt('migrateDocuments: db error'));
		1506	+ break;
		1507	+ }
		1508	+
		1509	+ $docs = count($result);
		1510	+ if ($docs == 0)
		1511	+ {
		1512	+ $noDocs = true;
		1513	+ break;
		1514	+ }
		1515	+ $numDocs += $docs;
		1516	+
		1517	+ foreach($result as $docinfo)
		1518	+ {
		1519	+ $docId = $docinfo['document_id'];
		1520	+
		1521	+ $document = Document::get($docId);
		1522	+ if (PEAR::isError($document) \|\| is_null($document))
		1523	+ {
		1524	+ $sql = "DELETE FROM document_text WHERE document_id=$docId";
		1525	+ DBUtil::runQuery($sql);
		1526	+ $default->log->error(sprintf(_kt('migrateDocuments: Could not get document %d\'s document! Removing content!'),$docId));
		1527	+ continue;
		1528	+ }
		1529	+
		1530	+ $version = $document->getMajorVersionNumber() . '.' . $document->getMinorVersionNumber();
		1531	+
		1532	+ $targetFile = tempnam($tempPath, 'ktindexer');
		1533	+
		1534	+ if (file_put_contents($targetFile, $docinfo['document_text']) === false)
		1535	+ {
		1536	+ $default->log->error(sprintf(_kt('migrateDocuments: Cannot write to \'%s\' for document id %d'), $targetFile, $docId));
		1537	+ continue;
		1538	+ }
		1539	+ // free memory asap ;)
		1540	+ unset($docinfo['document_text']);
		1541	+
		1542	+ $title = $document->getName();
		1543	+
		1544	+ $indexStatus = $this->indexDocumentAndDiscussion($docId, $targetFile, $title, $version);
		1545	+
		1546	+ if ($indexStatus)
		1547	+ {
		1548	+ $sql = "DELETE FROM document_text WHERE document_id=$docId";
		1549	+ DBUtil::runQuery($sql);
		1550	+ }
		1551	+ else
		1552	+ {
		1553	+ $default->log->error(sprintf(_kt("migrateDocuments: Problem indexing document %d"), $docId));
		1554	+ }
		1555	+
		1556	+ @unlink($targetFile);
		1557	+ }
		1558	+ }
		1559	+
		1560	+ @unlink($lockFile);
		1561	+
		1562	+ $time = KTUtil::getBenchmarkTime() - $start;
		1563	+
		1564	+ KTUtil::setSystemSetting('migrationTime', KTUtil::getSystemSetting('migrationTime',0) + $time);
		1565	+ KTUtil::setSystemSetting('migratedDocuments', KTUtil::getSystemSetting('migratedDocuments',0) + $numDocs);
		1566	+
		1567	+ $default->log->info(sprintf(_kt('migrateDocuments: stopping - done in %d seconds!'), $time));
		1568	+ if ($noDocs)
		1569	+ {
		1570	+ $default->log->info(_kt('migrateDocuments: Completed!'));
		1571	+ KTUtil::setSystemSetting('migrationComplete', 'true');
		1572	+ schedulerUtil::deleteByName('Index Migration');
		1573	+ $default->log->debug(_kt('migrateDocuments: Disabling \'Index Migration\' task by removing scheduler entry.'));
		1574	+ }
		1575	+ }
		1576	+
		1577	+ /**
		1578	+ * Index a document. The base class must override this function.
		1579	+ *
		1580	+ * @param int $docId
		1581	+ * @param string $textFile
		1582	+ */
		1583	+ protected abstract function indexDocument($docId, $textFile, $title, $version);
		1584	+
		1585	+
		1586	+ public function updateDocumentIndex($docId, $text)
		1587	+ {
		1588	+ $config = KTConfig::getSingleton();
		1589	+ $tempPath = $config->get("urls/tmpDirectory");
		1590	+ $tempFile = tempnam($tempPath,'ud_');
		1591	+
		1592	+ file_put_contents($tempFile, $text);
		1593	+
		1594	+ $document = Document::get($docId);
		1595	+ $title = $document->getDescription();
		1596	+ $version = $document->getVersion();
		1597	+
		1598	+ $result = $this->indexDocument($docId, $tempFile, $title, $version);
		1599	+
		1600	+ if (file_exists($tempFile))
		1601	+ {
		1602	+ unlink($tempFile);
		1603	+ }
		1604	+
		1605	+ return $result;
		1606	+ }
		1607	+
		1608	+ /**
		1609	+ * Index a discussion. The base class must override this function.
		1610	+ *
		1611	+ * @param int $docId
		1612	+ */
		1613	+ protected abstract function indexDiscussion($docId);
		1614	+
		1615	+ /**
		1616	+ * Diagnose the indexer. e.g. Check that the indexing server is running.
		1617	+ *
		1618	+ */
		1619	+ public abstract function diagnose();
		1620	+
		1621	+ /**
		1622	+ * Diagnose the extractors.
		1623	+ *
		1624	+ * @return array
		1625	+ */
		1626	+ public function diagnoseExtractors()
		1627	+ {
		1628	+ $diagnosis = $this->_diagnose($this->extractorPath, 'DocumentExtractor', 'Extractor.inc.php');
		1629	+ $diagnosis = array_merge($diagnosis, $this->_diagnose($this->hookPath, 'Hook', 'Hook.inc.php'));
		1630	+
		1631	+ return $diagnosis;
		1632	+ }
		1633	+
		1634	+ /**
		1635	+ * This is a refactored diagnose function.
		1636	+ *
		1637	+ * @param string $path
		1638	+ * @param string $class
		1639	+ * @param string $extension
		1640	+ * @return array
		1641	+ */
		1642	+ private function _diagnose($path, $baseclass, $extension)
		1643	+ {
		1644	+ global $default;
		1645	+
		1646	+ $diagnoses = array();
		1647	+
		1648	+ $dir = opendir(SearchHelper::correctPath($path));
		1649	+ $extlen = - strlen($extension);
		1650	+
		1651	+ while (($file = readdir($dir)) !== false)
		1652	+ {
		1653	+ if (substr($file,0,1) == '.')
		1654	+ {
		1655	+ continue;
		1656	+ }
		1657	+ if (substr($file,$extlen) != $extension)
		1658	+ {
		1659	+ $default->log->error(sprintf(_kt("diagnose: '%s' does not have extension '%s'."), $file, $extension));
		1660	+ continue;
		1661	+ }
		1662	+
		1663	+ require_once($path . '/' . $file);
		1664	+
		1665	+ $class = substr($file, 0, -8);
		1666	+ if (!class_exists($class))
		1667	+ {
		1668	+ $default->log->error(sprintf(_kt("diagnose: class '%s' does not exist."), $class));
		1669	+ continue;
		1670	+ }
		1671	+
		1672	+ if (!$this->isExtractorEnabled($class))
		1673	+ {
		1674	+ $default->log->debug(sprintf(_kt("diagnose: extractor '%s' is disabled."), $class));
		1675	+ continue;
		1676	+ }
		1677	+
		1678	+ $extractor = new $class();
		1679	+ if (!is_a($extractor, $baseclass))
		1680	+ {
		1681	+ $default->log->error(sprintf(_kt("diagnose(): '%s' is not of type DocumentExtractor"), $class));
		1682	+ continue;
		1683	+ }
		1684	+
		1685	+ $types = $extractor->getSupportedMimeTypes();
		1686	+ if (empty($types))
		1687	+ {
		1688	+ if ($this->debug) $default->log->debug(sprintf(_kt("diagnose: class '%s' does not support any types."), $class));
		1689	+ continue;
		1690	+ }
		1691	+
		1692	+ $diagnosis=$extractor->diagnose();
		1693	+ if (empty($diagnosis))
		1694	+ {
		1695	+ continue;
		1696	+ }
		1697	+ $diagnoses[$class] = array(
		1698	+ 'name'=>$extractor->getDisplayName(),
		1699	+ 'diagnosis'=>$diagnosis
		1700	+ );
		1701	+
		1702	+ }
		1703	+ closedir($dir);
		1704	+
		1705	+ return $diagnoses;
		1706	+ }
		1707	+
		1708	+
		1709	+ /**
		1710	+ * Register the extractor types.
		1711	+ *
		1712	+ * @param boolean $clear. Optional. Defaults to false.
		1713	+ */
		1714	+ public function registerTypes($clear=false)
		1715	+ {
		1716	+ if ($clear)
		1717	+ {
		1718	+ $this->clearExtractors();
		1719	+ }
		1720	+ $dir = opendir(SearchHelper::correctPath($this->extractorPath));
		1721	+ while (($file = readdir($dir)) !== false)
		1722	+ {
		1723	+ if (substr($file,-17) == 'Extractor.inc.php')
		1724	+ {
		1725	+ require_once($this->extractorPath . '/' . $file);
		1726	+ $class = substr($file, 0, -8);
		1727	+
		1728	+ if (!class_exists($class))
		1729	+ {
		1730	+ // if the class does not exist, we can't do anything.
		1731	+ continue;
		1732	+ }
		1733	+
		1734	+ $extractor = new $class;
		1735	+ if ($extractor instanceof DocumentExtractor)
		1736	+ {
		1737	+ $extractor->registerMimeTypes();
		1738	+ }
		1739	+ }
		1740	+ }
		1741	+ closedir($dir);
		1742	+ }
		1743	+
		1744	+ /**
		1745	+ * This is used as a possible obtimisation effort. It may be overridden in that case.
		1746	+ *
		1747	+ * @param int $docId
		1748	+ * @param string $textFile
		1749	+ */
		1750	+ protected function indexDocumentAndDiscussion($docId, $textFile, $title, $version)
		1751	+ {
		1752	+ $this->indexDocument($docId, $textFile, $title, $version);
		1753	+ $this->indexDiscussion($docId);
		1754	+ }
		1755	+
		1756	+ /**
		1757	+ * Remove the document from the queue. This is normally called when it has been processed.
		1758	+ *
		1759	+ * @param int $docid
		1760	+ */
		1761	+ public static function unqueueDocument($docid, $reason=false, $level='debug')
		1762	+ {
		1763	+ $sql = "DELETE FROM index_files WHERE document_id=$docid";
		1764	+ DBUtil::runQuery($sql);
		1765	+ if ($reason !== false)
		1766	+ {
		1767	+ global $default;
		1768	+ $default->log->$level("Indexer: removing document $docid from the queue - $reason");
		1769	+ }
		1770	+ }
		1771	+
		1772	+ /**
		1773	+ * Run a query on the index.
		1774	+ *
		1775	+ * @param string $query
		1776	+ * @return array
		1777	+ */
		1778	+ public abstract function query($query);
		1779	+
		1780	+ /**
		1781	+ * Converts an integer to a string that can be easily compared and reversed.
		1782	+ *
		1783	+ * @param int $int
		1784	+ * @return string
		1785	+ */
		1786	+ public static function longToString($int)
		1787	+ {
		1788	+ $maxlen = 14;
		1789	+
		1790	+ $a2z = array('a','b','c','d','e','f','g','h','i','j');
		1791	+ $o29 = array('0','1','2','3','4','5','6','7','8','9');
		1792	+ $l = str_pad('',$maxlen - strlen("$int"),'0') . $int;
		1793	+
		1794	+ return str_replace($o29, $a2z, $l);
		1795	+ }
		1796	+
		1797	+ /**
		1798	+ * Converts a string to an integer.
		1799	+ *
		1800	+ * @param string $str
		1801	+ * @return int
		1802	+ */
		1803	+ public static function stringToLong($str)
		1804	+ {
		1805	+ $a2z = array('a','b','c','d','e','f','g','h','i','j');
		1806	+ $o29 = array('0','1','2','3','4','5','6','7','8','9');
		1807	+
		1808	+ $int = str_replace($a2z, $o29, $str) + 0;
		1809	+
		1810	+ return $int;
		1811	+ }
		1812	+
		1813	+ /**
		1814	+ * Possibly we can optimise indexes. This method must be overriden.
		1815	+ * The new function must call the parent!
		1816	+ *
		1817	+ */
		1818	+ public function optimise()
		1819	+ {
		1820	+ KTUtil::setSystemSetting('luceneOptimisationDate', time());
		1821	+ }
		1822	+
		1823	+ /**
		1824	+ * Shuts down the indexer
		1825	+ *
		1826	+ */
		1827	+ public function shutdown()
		1828	+ {
		1829	+ // do nothing generally
		1830	+ }
		1831	+
		1832	+ /**
		1833	+ * Returns the name of the indexer.
		1834	+ *
		1835	+ * @return string
		1836	+ */
		1837	+ public abstract function getDisplayName();
		1838	+
		1839	+
		1840	+ /**
		1841	+ * Returns the number of non-deleted documents in the index.
		1842	+ *
		1843	+ * @return int
		1844	+ */
		1845	+ public abstract function getDocumentsInIndex();
		1846	+
		1847	+ public abstract function isDocumentIndexed($documentId);
		1848	+
		1849	+ /**
		1850	+ * Returns the path to the index directory
		1851	+ *
		1852	+ * @return string
		1853	+ */
		1854	+ public function getIndexDirectory()
		1855	+ {
		1856	+ $config = KTConfig::getSingleton();
		1857	+ $directory = $config->get('indexer/luceneDirectory');
		1858	+ return $directory;
		1859	+ }
		1860	+}
		1861	+
		1862	+?>