From ce268477f8cab496a2d185131fe8ecde46a90604 Mon Sep 17 00:00:00 2001 From: Megan Date: Fri, 10 Apr 2009 10:49:48 +0200 Subject: [PATCH] KTS-4140. Moved the bulk download into a background task. Changed it to use PEAR archive functionality instead of the zip binary. "CLONE -error: The ZIP file can only be downloaded once - if you cancel the download, you will need to reload the page. (SUP-1680)" In progress. --- lib/foldermanagement/compressionArchiveUtil.inc.php | 527 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------- lib/foldermanagement/downloadTask.php | 47 +++++++++++++++++++++++++++++++++++++++++++++++ plugins/ktcore/KTBulkActions.php | 160 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------------------------------------------------------------------------------------- plugins/ktstandard/KTBulkExportPlugin.php | 45 +++++++++++++++++++++++++++++++++------------ sql/mysql/install/data.sql | 7 +++++-- sql/mysql/install/structure.sql | 16 ++++++++++++++++ sql/mysql/upgrade/3.6/download_queue.sql | 23 +++++++++++++++++++++++ templates/ktcore/action/bulk_download.smarty | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 715 insertions(+), 157 deletions(-) create mode 100644 lib/foldermanagement/downloadTask.php create mode 100644 sql/mysql/upgrade/3.6/download_queue.sql create mode 100644 templates/ktcore/action/bulk_download.smarty diff --git a/lib/foldermanagement/compressionArchiveUtil.inc.php b/lib/foldermanagement/compressionArchiveUtil.inc.php index d38aaa3..4dbba01 100644 --- a/lib/foldermanagement/compressionArchiveUtil.inc.php +++ b/lib/foldermanagement/compressionArchiveUtil.inc.php @@ -36,6 +36,8 @@ * */ +require_once('File/Archive.php'); + /** * Class to create and download a zip file */ @@ -50,6 +52,8 @@ class ZipFolder { var $aReplaceKeys = array(); var $aReplaceValues = array(); var $sOutputEncoding = 'UTF-8'; + var $extension = 'zip'; + var $exportCode = null; /** * Constructor @@ -57,22 +61,22 @@ class ZipFolder { * @param string $sZipFileName The name of the zip file - gets ignored at the moment. * @param string $exportCode The code to use if a zip file has already been created. */ - function ZipFolder($sZipFileName, $exportCode = null) { + function ZipFolder($sZipFileName = null, $exportCode = null, $extension = 'zip') { $this->oKTConfig =& KTConfig::getSingleton(); $this->oStorage =& KTStorageManagerUtil::getSingleton(); $this->sOutputEncoding = $this->oKTConfig->get('export/encoding', 'UTF-8'); + $this->extension = $extension; $this->sPattern = "[\*|\%|\\\|\/|\<|\>|\+|\:|\?|\||\'|\"]"; $this->sFolderPattern = "[\*|\%|\<|\>|\+|\:|\?|\||\'|\"]"; - // If the export code exists then a temp zip directory has already been created - if(!empty($exportCode)){ - $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode); + $this->exportCode = $exportCode; - if(!empty($aData)){ - $sTmpPath = $aData['dir']; - } + // Check if the temp directory has been created and stored in session + $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode); + if(!empty($aData) && isset($aData['dir'])){ + $sTmpPath = $aData['dir']; }else { $sBasedir = $this->oKTConfig->get("urls/tmpDirectory"); $sTmpPath = tempnam($sBasedir, 'kt_compress_zip'); @@ -100,6 +104,15 @@ class ZipFolder { $this->aReplaceValues = array_values($aReplace); } + static public function get($exportCode) + { + static $zipFolder = null; + if(is_null($zipFolder)){ + $zipFolder = new ZipFolder('', $exportCode); + } + return $zipFolder; + } + /** * Return the full path * @@ -154,7 +167,7 @@ class ZipFolder { $sOrigFile = $this->oStorage->temporaryFile($oDocument); $sFilename = $sParentFolder.'/'.$sDocName; - copy($sOrigFile, $sFilename); + @copy($sOrigFile, $sFilename); $this->aPaths[] = $sDocPath.'/'.$sDocName; return true; @@ -190,49 +203,62 @@ class ZipFolder { return PEAR::raiseError(_kt("No folders or documents found to compress")); } + $config = KTConfig::getSingleton(); + $useBinary = $config->get('export/useBinary', false); + // Set environment language to output character encoding $loc = $this->sOutputEncoding; putenv("LANG=$loc"); putenv("LANGUAGE=$loc"); $loc = setlocale(LC_ALL, $loc); + if($useBinary){ + $sManifest = sprintf("%s/%s", $this->sTmpPath, "MANIFEST"); + file_put_contents($sManifest, join("\n", $this->aPaths)); + } - - $sManifest = sprintf("%s/%s", $this->sTmpPath, "MANIFEST"); - file_put_contents($sManifest, join("\n", $this->aPaths)); - $sZipFile = sprintf("%s/%s.zip", $this->sTmpPath, $this->sZipFileName); + $sZipFile = sprintf("%s/%s.".$this->extension, $this->sTmpPath, $this->sZipFileName); $sZipFile = str_replace('<', '', str_replace('', '', $sZipFile))); - $sZipCommand = KTUtil::findCommand("export/zip", "zip"); - $aCmd = array($sZipCommand, "-r", $sZipFile, ".", "-i@MANIFEST"); - $sOldPath = getcwd(); - chdir($this->sTmpPath); - - // Note that the popen means that pexec will return a file descriptor - $aOptions = array('popen' => 'r'); - $fh = KTUtil::pexec($aCmd, $aOptions); - - if($bEchoStatus){ - $last_beat = time(); - while(!feof($fh)) { - if ($i % 1000 == 0) { - $this_beat = time(); - if ($last_beat + 1 < $this_beat) { - $last_beat = $this_beat; - print " "; + + if($useBinary){ + $sZipCommand = KTUtil::findCommand("export/zip", "zip"); + $aCmd = array($sZipCommand, "-r", $sZipFile, ".", "-i@MANIFEST"); + $sOldPath = getcwd(); + chdir($this->sTmpPath); + + // Note that the popen means that pexec will return a file descriptor + $aOptions = array('popen' => 'r'); + $fh = KTUtil::pexec($aCmd, $aOptions); + + if($bEchoStatus){ + $last_beat = time(); + while(!feof($fh)) { + if ($i % 1000 == 0) { + $this_beat = time(); + if ($last_beat + 1 < $this_beat) { + $last_beat = $this_beat; + print " "; + } } + $contents = fread($fh, 4096); + if ($contents) { + print nl2br($this->_convertEncoding($contents, false)); + } + $i++; } - $contents = fread($fh, 4096); - if ($contents) { - print nl2br($this->_convertEncoding($contents, false)); - } - $i++; } + pclose($fh); + }else{ + // Create the zip archive using the PEAR File_Archive + File_Archive::extract( + File_Archive::read($this->sTmpPath.'/Root Folder'), + File_Archive::toArchive($this->sZipFileName.'.'.$this->extension, File_Archive::toFiles($this->sTmpPath), $this->extension) + ); } - pclose($fh); // Save the zip file and path into session $_SESSION['zipcompression'] = KTUtil::arrayGet($_SESSION, 'zipcompression', array()); - $sExportCode = KTUtil::randomString(); + $sExportCode = $this->exportCode; $_SESSION['zipcompression'][$sExportCode] = array( 'file' => $sZipFile, 'dir' => $this->sTmpPath, @@ -262,18 +288,41 @@ class ZipFolder { } if (!file_exists($sZipFile)) { - return PEAR::raiseError(_kt('The ZIP file can only be downloaded once - if you cancel the download, you will need to reload the page.')); + return PEAR::raiseError(_kt('The zip file has not been created, if you are downloading a large number of documents + or a large document then it may take a few seconds to finish. Try refreshing the page.')); } $mimeType = 'application/zip; charset=utf-8;'; $fileSize = filesize($sZipFile); - $fileName = $this->sZipFileName . '.zip'; + $fileName = $this->sZipFileName .'.'.$this->extension; KTUtil::download($sZipFile, $mimeType, $fileSize, $fileName); KTUtil::deleteDirectory($sTmpPath); return true; } + function checkArchiveExists($exportCode = null) + { + if(!(isset($exportCode) && !empty($exportCode))) { + $exportCode = KTUtil::arrayGet($_SESSION['zipcompression'], 'exportcode'); + } + + $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode); + + if(!empty($aData)){ + $sZipFile = $aData['file']; + $sTmpPath = $aData['dir']; + }else{ + $sZipFile = $this->sZipFile; + $sTmpPath = $this->sTmpPath; + } + + if (!file_exists($sZipFile)) { + return false; + } + return true; + } + /** * Check that iconv exists and that the selected encoding is supported. */ @@ -301,5 +350,407 @@ class ZipFolder { return iconv($this->sOutputEncoding, "UTF-8", $sMystring); } } + + static public function checkDownloadSize($object) + { + return true; + + if($object instanceof Document || $object instanceof DocumentProxy){ + } + + if($object instanceof Folder || $object instanceof FolderProxy){ + $id = $object->iId; + + // If we're working with the root folder + if($id = 1){ + $sql = 'SELECT count(*) as cnt FROM documents where folder_id = 1'; + }else{ + $sql[] = "SELECT count(*) as cnt FROM documents where parent_folder_ids like '%,?' OR parent_folder_ids like '%,?,%' OR folder_id = ?"; + $sql[] = array($id, $id, $id); + } + + /* + SELECT count(*) FROM documents d + INNER JOIN document_metadata_version m ON d.metadata_version_id = m.id + INNER JOIN document_content_version c ON m.content_version_id = c.id + where (d.parent_folder_ids like '%,12' OR d.parent_folder_ids like '%,12,%' OR d.folder_id = 12) AND d.status_id < 3 AND size > 100000 + */ + + $result = DBUtil::getOneResult($sql); + + if($result['cnt'] > 10){ + return true; + } + } + + return false; + } +} + +/** + * Class to manage the queue of bulk downloads + * + */ +class DownloadQueue +{ + private $bNoisy; + private $bNotifications; + private $errors; + + public function __construct() + { + $config = KTConfig::getSingleton(); + $this->bNoisy = $config->get('tweaks/noisyBulkOperations', false); + $this->bNotifications = ($config->get('export/enablenotifications', 'on') == 'on') ? true : false; + } + + /** + * Add an item to the download queue + * + * @param string $code The identification string for the download + * @param string $id The object id + * @param string $type The type of object Folder | Document + */ + static public function addItem($code, $folderId, $id, $type) + { + $fields = array(); + $fields['code'] = $code; + $fields['folder_id'] = $folderId; + $fields['object_id'] = $id; + $fields['object_type'] = $type; + $fields['user_id'] = $_SESSION['userID']; + $fields['date_added'] = date('Y-m-d H:i:s'); + + $res = DBUtil::autoInsert('download_queue', $fields); + } + + public function removeItem($code) + { + $where = array('code' => $code); + $res = DBUtil::whereDelete('download_queue', $where); + return $res; + } + + public function getQueue() + { + $sql = 'SELECT * FROM download_queue d ORDER BY date_added, code'; + $rows = DBUtil::getResultArray($sql); + + if(PEAR::isError($rows)){ + return $rows; + } + + $queue = array(); + foreach ($rows as $item){ + $queue[$item['code']][] = $item; + } + return $queue; + } + + public function getItemStatus($code) + { + $sql = array(); + $sql[] = 'SELECT status, errors FROM download_queue WHERE code = ?'; + $sql[] = $code; + $result = DBUtil::getResultArray($sql); + return $result; + } + + public function setItemStatus($code, $status = 1, $error = null) + { + $fields = array(); + $fields['status'] = $status; + $fields['errors'] = !empty($error) ? json_encode($error) : null; + $where = array('code' => $code); + $res = DBUtil::whereUpdate('download_queue', $fields, $where); + return $res; + } + + public function processQueue() + { + global $default; + + // get items from queue + $queue = $this->getQueue(); + if(PEAR::isError($queue)){ + $default->log->debug('Download Queue: error on fetching queue - '.$queue->getMessage()); + return false; + } + + // Loop through items and create downloads + foreach ($queue as $code => $download){ + // reset the error messages + $this->errors = null; + + // if the user_id is not set then skip + if(!isset($download[0]['user_id']) || empty($download[0]['user_id'])){ + $default->log->debug('Download Queue: no user id set for download code '.$code); + $error = array(_kt('No user id has been set, the archive cannot be created.')); + $result = $this->setItemStatus($code, 2, $error); + continue; + } + + // Force a session for the user + $_SESSION['userID'] = $download[0]['user_id']; + $baseFolderId = $download[0]['folder_id']; + + // Create a new instance of the archival class + $zip = new ZipFolder('', $code); + $res = $zip->checkConvertEncoding(); + + if(PEAR::isError($res)){ + $default->log->error('Download Queue: Archive class check convert encoding error - '.$res->getMessage()); + $error = array(_kt('The archive cannot be created. An error occurred in the encoding.')); + $result = $this->setItemStatus($code, 2, $error); + continue; + } + + $default->log->debug('Download Queue: Creating download for user: '.$_SESSION['userID'].', code: '.$code); + + DBUtil::startTransaction(); + + // Add the individual files and folders into the archive + foreach ($download as $item){ + if($item['object_type'] == 'document'){ + $docId = $item['object_id']; + $this->addDocument($zip, $docId); + } + if($item['object_type'] == 'folder'){ + $folderId = $item['object_id']; + $this->addFolder($zip, $folderId); + } + } + + $res = $zip->createZipFile(); + + if(PEAR::isError($res)){ + $default->log->debug('Download Queue: Archive could not be created. Exiting transaction. '.$res->getMessage()); + DBUtil::rollback(); + + $error = array(_kt('The archive could not be created.')); + $result = $this->setItemStatus($code, 2, $error); + continue; + } + + $default->log->debug('Download Queue: Archival successful'); + + $oTransaction = KTFolderTransaction::createFromArray(array( + 'folderid' => $baseFolderId, + 'comment' => "Bulk export", + 'transactionNS' => 'ktstandard.transactions.bulk_export', + 'userid' => $_SESSION['userID'], + 'ip' => Session::getClientIP(), + )); + + if(PEAR::isError($oTransaction)){ + $default->log->debug('Download Queue: transaction could not be logged. '.$oTransaction->getMessage()); + } + + DBUtil::commit(); + + // Set status for the download + $this->errors['archive'] = $_SESSION['zipcompression']; + $result = $this->setItemStatus($code, 1, $this->errors); + if(PEAR::isError($result)){ + $default->log->error('Download Queue: item status could not be set for user: '.$_SESSION['userID'].', code: '.$code.', error: '.$result->getMessage()); + } + // reset the error messages + $this->errors = null; + $_SESSION['zipcompression'] = null; + } + } + + public function addDocument(&$zip, $docId) + { + + $oDocument = Document::get($docId); + if(PEAR::isError($oDocument)){ + $this->errors[] = _kt('Document cannot be exported, an error occurred: ').$oDocument->getMessage(); + return $oDocument; + } + + if ($this->bNoisy) { + $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array()); + $oDocumentTransaction->create(); + } + + // fire subscription alerts for the downloaded document - if global config is set + if($this->bNotifications){ + $oSubscriptionEvent = new SubscriptionEvent(); + $oFolder = Folder::get($oDocument->getFolderID()); + $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder); + } + + return $zip->addDocumentToZip($oDocument); + } + + public function addFolder(&$zip, $folderId) + { + $oFolder = Folder::get($folderId); + + if(PEAR::isError($oFolder)){ + $this->errors[] = _kt('Folder cannot be exported, an error occurred: ').$oFolder->getMessage(); + return $oFolder; + } + + $sFolderDocs = $oFolder->getDocumentIDs($folderId); + if(PEAR::isError($sFolderDocs)){ + $default->log->error('Download Queue: get document ids for folder caused an error: '.$sFolderDocs->getMessage()); + $sFolderDocs = ''; + } + + // Add folder to zip + $zip->addFolderToZip($oFolder); + + $aDocuments = array(); + if(!empty($sFolderDocs)){ + $aDocuments = explode(',', $sFolderDocs); + } + + // Get all the folders within the current folder + $sWhereClause = "parent_folder_ids like '%,{$folderId}' + OR parent_folder_ids like '%,{$folderId},%' + OR parent_id = {$folderId}"; + + $aFolderList = $oFolder->getList($sWhereClause); + $aLinkingFolders = $this->getLinkingEntities($aFolderList); + $aFolderList = array_merge($aFolderList,$aLinkingFolders); + + $aFolderObjects = array(); + $aFolderObjects[$folderId] = $oFolder; + + // Export the folder structure to ensure the export of empty directories + if(!empty($aFolderList)){ + foreach($aFolderList as $k => $oFolderItem){ + if($oFolderItem->isSymbolicLink()){ + $oFolderItem = $oFolderItem->getLinkedFolder(); + } + if(Permission::userHasFolderReadPermission($oFolderItem)){ + // Get documents for each folder + $sFolderItemId = $oFolderItem->getID(); + $sFolderItemDocs = $oFolderItem->getDocumentIDs($sFolderItemId); + + if(!empty($sFolderItemDocs)){ + $aFolderDocs = explode(',', $sFolderItemDocs); + $aDocuments = array_merge($aDocuments, $aFolderDocs); + } + $zip->addFolderToZip($oFolderItem); + $aFolderObjects[$oFolderItem->getId()] = $oFolderItem; + } + } + } + + // Add all documents to the export + if(!empty($aDocuments)){ + foreach($aDocuments as $sDocumentId){ + $oDocument = Document::get($sDocumentId); + if($oDocument->isSymbolicLink()){ + $oDocument->switchToLinkedCore(); + } + if(Permission::userHasDocumentReadPermission($oDocument)){ + + if(!KTWorkflowUtil::actionEnabledForDocument($oDocument, 'ktcore.actions.document.view')){ + $this->errors[] = $oDocument->getName().': '._kt('Document cannot be exported as it is restricted by the workflow.'); + continue; + } + + $sDocFolderId = $oDocument->getFolderID(); + $oFolder = isset($aFolderObjects[$sDocFolderId]) ? $aFolderObjects[$sDocFolderId] : Folder::get($sDocFolderId); + + if ($this->bNoisy) { + $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array()); + $oDocumentTransaction->create(); + } + + // fire subscription alerts for the downloaded document + if($this->bNotifications){ + $oSubscriptionEvent = new SubscriptionEvent(); + $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder); + } + + $zip->addDocumentToZip($oDocument, $oFolder); + } + } + } + } + + function getLinkingEntities($aFolderList){ + $aSearchFolders = array(); + if(!empty($aFolderList)){ + foreach($aFolderList as $oFolderItem){ + if(Permission::userHasFolderReadPermission($oFolderItem)){ + // If it is a shortcut, we should do some more searching + if($oFolderItem->isSymbolicLink()){ + $oFolderItem = $oFolderItem->getLinkedFolder(); + $aSearchFolders[] = $oFolderItem->getID(); + } + } + } + } + $aLinkingFolders = array(); + $aSearchCompletedFolders = array(); + $count = 0; + while(count($aSearchFolders)>0){ + $count++; + $oFolder = Folder::get(array_pop($aSearchFolders)); + $folderId = $oFolder->getId(); + // Get all the folders within the current folder + $sWhereClause = "parent_folder_ids = '{$folderId}' OR + parent_folder_ids LIKE '{$folderId},%' OR + parent_folder_ids LIKE '%,{$folderId},%' OR + parent_folder_ids LIKE '%,{$folderId}'"; + $aFolderList = $this->oFolder->getList($sWhereClause); + foreach($aFolderList as $oFolderItem){ + if($oFolderItem->isSymbolicLink()){ + $oFolderItem = $oFolderItem->getLinkedFolder(); + } + if(Permission::userHasFolderReadPermission($oFolderItem)){ + if($aSearchCompletedFolders[$oFolderItem->getID()] != true){ + $aSearchFolders[] = $oFolderItem->getID(); + $aSearchCompletedFolders[$oFolderItem->getID()] = true; + } + } + } + if(!isset($aLinkingFolders[$oFolder->getId()])){ + $aLinkingFolders[$oFolder->getId()] = $oFolder; + } + } + return $aLinkingFolders; + } + + public function isDownloadAvailable($code) + { + $check = $this->getItemStatus($code); + $status = $check[0]['status']; + + if($status < 1){ + return false; + } + + $message = $check[0]['errors']; + $message = json_decode($message, true); + + if($status > 1){ + return $message; + } + + // Create the archive session variables + $_SESSION['zipcompression'] = $message['archive']; + unset($message['archive']); + + // Check that the archive has been created + $zip = new ZipFolder('', $code); + if($zip->checkArchiveExists($code)){ + // Clean up the download queue and return errors + $this->removeItem($code); + return $message; + } + return false; + } + + public function isLocked() + { + return false; + } } ?> diff --git a/lib/foldermanagement/downloadTask.php b/lib/foldermanagement/downloadTask.php new file mode 100644 index 0000000..cd33cbb --- /dev/null +++ b/lib/foldermanagement/downloadTask.php @@ -0,0 +1,47 @@ +isDownloadAvailable($code); + + if($status === false){ + echo 'wait'; + }else{ + $str = ''; + // display any error messages + if(!empty($status)){ + $str = '
'._kt('The following errors occurred during the download').':
'; + $str .= ''; + foreach ($status as $msg){ + $str .= ''; + } + $str .= '
'.$msg.'
'; + } + echo $str; + } + exit(0); +} + +if($queue->isLocked()){ + exit(0); +} +// Not a ping, process the queue +$queue->processQueue(); + +exit(0); +?> \ No newline at end of file diff --git a/plugins/ktcore/KTBulkActions.php b/plugins/ktcore/KTBulkActions.php index bd6c5cf..2a04c5b 100644 --- a/plugins/ktcore/KTBulkActions.php +++ b/plugins/ktcore/KTBulkActions.php @@ -410,12 +410,12 @@ class KTBulkMoveAction extends KTBulkAction { if(is_a($oEntity, 'Folder')) { $aDocuments = array(); $aChildFolders = array(); - + $oFolder = $oEntity; // Get folder id $sFolderId = $oFolder->getID(); - + // Get documents in folder $sDocuments = $oFolder->getDocumentIDs($sFolderId); $aDocuments = (!empty($sDocuments)) ? explode(',', $sDocuments) : array(); @@ -645,7 +645,7 @@ class KTBulkCopyAction extends KTBulkAction { return PEAR::raiseError(_kt('Document cannot be copied')); } } - + if(is_a($oEntity, 'Folder')) { $aDocuments = array(); $aChildFolders = array(); @@ -1068,18 +1068,42 @@ class KTBrowseBulkExportAction extends KTBulkAction { return PEAR::raiseError(_kt('Document cannot be exported as it is restricted by the workflow.')); } } + return parent::check_entity($oEntity); } - function do_performaction() { - $folderName = $this->oFolder->getName(); - $this->oZip = new ZipFolder($folderName); - $res = $this->oZip->checkConvertEncoding(); + $config = KTConfig::getSingleton(); + $useQueue = $config->get('export/useDownloadQueue', true); + + // Create the export code + $this->sExportCode = KTUtil::randomString(); + $_SESSION['exportcode'] = $this->sExportCode; $folderurl = $this->getReturnUrl(); $sReturn = sprintf('

' . _kt('Return to the original folder') . "

\n", $folderurl); + $download_url = KTUtil::addQueryStringSelf("action=downloadZipFile&fFolderId={$this->oFolder->getId()}&exportcode={$this->sExportCode}"); + + if($useQueue){ + $result = parent::do_performaction(); + + $url = KTUtil::kt_url() . '/lib/foldermanagement/downloadTask.php'; + + $oTemplating =& KTTemplating::getSingleton(); + $oTemplate = $oTemplating->loadTemplate('ktcore/action/bulk_download'); + + $aParams = array( + 'return' => $sReturn, + 'url' => $url, + 'code' => $this->sExportCode, + 'download_url' => $download_url + ); + return $oTemplate->render($aParams); + } + + $this->oZip = new ZipFolder('', $this->sExportCode); + $res = $this->oZip->checkConvertEncoding(); if(PEAR::isError($res)){ $this->addErrorMessage($res->getMessage()); @@ -1087,16 +1111,14 @@ class KTBrowseBulkExportAction extends KTBulkAction { } $this->startTransaction(); - $oKTConfig =& KTConfig::getSingleton(); - $this->bNoisy = $oKTConfig->get("tweaks/noisyBulkOperations"); - - $this->bNotifications = ($oKTConfig->get('export/enablenotifications', 'on') == 'on') ? true : false; $result = parent::do_performaction(); + $sExportCode = $this->oZip->createZipFile(); if(PEAR::isError($sExportCode)){ $this->addErrorMessage($sExportCode->getMessage()); + $this->rollbackTransaction(); return $sReturn; } @@ -1110,23 +1132,28 @@ class KTBrowseBulkExportAction extends KTBulkAction { $this->commitTransaction(); - $url = KTUtil::addQueryStringSelf(sprintf('action=downloadZipFile&fFolderId=%d&exportcode=%s', $this->oFolder->getId(), $sExportCode)); - $str = sprintf('

' . _kt('Your download will begin shortly. If you are not automatically redirected to your download, please click here ') . "

\n", $url); + $str = sprintf('

' . _kt('Your download will begin shortly. If you are not automatically redirected to your download, please click here ') . "

\n", $download_url); $str .= sprintf('

' . _kt('Once your download is complete, click here to return to the original folder') . "

\n", $folderurl); - //$str .= sprintf("\n"); $str .= sprintf('', $url); + callLater(5, kt_bulkexport_redirect); + ', $download_url); return $str; } function perform_action($oEntity) { + $exportCode = $_SESSION['exportcode']; + $this->oZip = ZipFolder::get($exportCode); + + $oQueue = new DownloadQueue(); + + $config = KTConfig::getSingleton(); + $useQueue = $config->get('export/useDownloadQueue'); + if(is_a($oEntity, 'Document')) { $oDocument = $oEntity; @@ -1134,19 +1161,12 @@ class KTBrowseBulkExportAction extends KTBulkAction { $oDocument->switchToLinkedCore(); } - if ($this->bNoisy) { - $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array()); - $oDocumentTransaction->create(); - } - - // fire subscription alerts for the downloaded document - if global config is set - if($this->bNotifications){ - $oSubscriptionEvent = new SubscriptionEvent(); - $oFolder = Folder::get($oDocument->getFolderID()); - $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder); - } + if($useQueue){ + DownloadQueue::addItem($this->sExportCode, $this->oFolder->getId(), $oDocument->iId, 'document'); + }else{ + $oQueue->addDocument($this->oZip, $oDocument->iId); + } - $this->oZip->addDocumentToZip($oDocument); }else if(is_a($oEntity, 'Folder')) { $aDocuments = array(); @@ -1156,79 +1176,11 @@ class KTBrowseBulkExportAction extends KTBulkAction { $oFolder = $oFolder->getLinkedFolder(); } $sFolderId = $oFolder->getId(); - $sFolderDocs = $oFolder->getDocumentIDs($sFolderId); - - // Add folder to zip - $this->oZip->addFolderToZip($oFolder); - if(!empty($sFolderDocs)){ - $aDocuments = explode(',', $sFolderDocs); - } - - // Get all the folders within the current folder - $sWhereClause = "parent_folder_ids = '{$sFolderId}' OR - parent_folder_ids LIKE '{$sFolderId},%' OR - parent_folder_ids LIKE '%,{$sFolderId},%' OR - parent_folder_ids LIKE '%,{$sFolderId}'"; - $aFolderList = $this->oFolder->getList($sWhereClause); - $aLinkingFolders = $this->getLinkingEntities($aFolderList); - $aFolderList = array_merge($aFolderList,$aLinkingFolders); - - $aFolderObjects = array(); - $aFolderObjects[$sFolderId] = $oFolder; - - // Export the folder structure to ensure the export of empty directories - if(!empty($aFolderList)){ - foreach($aFolderList as $k => $oFolderItem){ - if($oFolderItem->isSymbolicLink()){ - $oFolderItem = $oFolderItem->getLinkedFolder(); - } - if(Permission::userHasFolderReadPermission($oFolderItem)){ - // Get documents for each folder - $sFolderItemId = $oFolderItem->getID(); - $sFolderItemDocs = $oFolderItem->getDocumentIDs($sFolderItemId); - - if(!empty($sFolderItemDocs)){ - $aFolderDocs = explode(',', $sFolderItemDocs); - $aDocuments = array_merge($aDocuments, $aFolderDocs); - } - $this->oZip->addFolderToZip($oFolderItem); - $aFolderObjects[$oFolderItem->getId()] = $oFolderItem; - } - } - } - - // Add all documents to the export - if(!empty($aDocuments)){ - foreach($aDocuments as $sDocumentId){ - $oDocument = Document::get($sDocumentId); - if($oDocument->isSymbolicLink()){ - $oDocument->switchToLinkedCore(); - } - if(Permission::userHasDocumentReadPermission($oDocument)){ - - if(!KTWorkflowUtil::actionEnabledForDocument($oDocument, 'ktcore.actions.document.view')){ - $this->addErrorMessage($oDocument->getName().': '._kt('Document cannot be exported as it is restricted by the workflow.')); - continue; - } - - $sDocFolderId = $oDocument->getFolderID(); - $oFolder = isset($aFolderObjects[$sDocFolderId]) ? $aFolderObjects[$sDocFolderId] : Folder::get($sDocFolderId); - - if ($this->bNoisy) { - $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array()); - $oDocumentTransaction->create(); - } - - // fire subscription alerts for the downloaded document - if($this->bNotifications){ - $oSubscriptionEvent = new SubscriptionEvent(); - $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder); - } - - $this->oZip->addDocumentToZip($oDocument, $oFolder); - } - } + if($useQueue){ + DownloadQueue::addItem($this->sExportCode, $this->oFolder->getId(), $sFolderId, 'folder'); + }else{ + $oQueue->addFolder($this->oZip, $sFolderId); } } return true; @@ -1237,9 +1189,7 @@ class KTBrowseBulkExportAction extends KTBulkAction { function do_downloadZipFile() { $sCode = $this->oValidator->validateString($_REQUEST['exportcode']); - $folderName = $this->oFolder->getName(); - $this->oZip = new ZipFolder($folderName, $sCode); - + $this->oZip = new ZipFolder('', $sCode); $res = $this->oZip->downloadZipFile($sCode); if(PEAR::isError($res)){ diff --git a/plugins/ktstandard/KTBulkExportPlugin.php b/plugins/ktstandard/KTBulkExportPlugin.php index e349d58..9148c90 100644 --- a/plugins/ktstandard/KTBulkExportPlugin.php +++ b/plugins/ktstandard/KTBulkExportPlugin.php @@ -71,20 +71,44 @@ class KTBulkExportAction extends KTFolderAction { } function do_main() { - $folderName = $this->oFolder->getName(); - $this->oZip = new ZipFolder($folderName); + $config = KTConfig::getSingleton(); + $useQueue = $config->get('export/useDownloadQueue', true); + + // Create the export code + $exportCode = KTUtil::randomString(); + $this->oZip = new ZipFolder('', $exportCode); if(!$this->oZip->checkConvertEncoding()) { redirect(KTBrowseUtil::getUrlForFolder($this->oFolder)); exit(0); } - $oKTConfig =& KTConfig::getSingleton(); - $bNoisy = $oKTConfig->get("tweaks/noisyBulkOperations"); - $bNotifications = ($oKTConfig->get('export/enablenotifications', 'on') == 'on') ? true : false; + $bNoisy = $config->get("tweaks/noisyBulkOperations"); + $bNotifications = ($config->get('export/enablenotifications', 'on') == 'on') ? true : false; - // Get all folders and sub-folders $sCurrentFolderId = $this->oFolder->getId(); + $url = KTUtil::addQueryStringSelf(sprintf('action=downloadZipFile&fFolderId=%d&exportcode=%s', $sCurrentFolderId, $exportCode)); + $folderurl = KTBrowseUtil::getUrlForFolder($this->oFolder); + $sReturn = '

' . _kt('Once your download is complete, click here to return to the original folder') . "

\n"; + + if($useQueue){ + DownloadQueue::addItem($exportCode, $sCurrentFolderId, $sCurrentFolderId, 'folder'); + + $task_url = KTUtil::kt_url() . '/lib/foldermanagement/downloadTask.php'; + + $oTemplating =& KTTemplating::getSingleton(); + $oTemplate = $oTemplating->loadTemplate('ktcore/action/bulk_download'); + + $aParams = array( + 'return' => $sReturn, + 'url' => $task_url, + 'code' => $exportCode, + 'download_url' => $url + ); + return $oTemplate->render($aParams); + } + + // Get all folders and sub-folders $sWhereClause = "parent_folder_ids = '{$sCurrentFolderId}' OR parent_folder_ids LIKE '{$sCurrentFolderId},%' OR parent_folder_ids LIKE '%,{$sCurrentFolderId},%' OR @@ -173,16 +197,14 @@ class KTBulkExportAction extends KTFolderAction { 'ip' => Session::getClientIP(), )); - $url = KTUtil::addQueryStringSelf(sprintf('action=downloadZipFile&fFolderId=%d&exportcode=%s', $this->oFolder->getId(), $sExportCode)); printf('

' . _kt('Your download will begin shortly. If you are not automatically redirected to your download, please click here ') . "

\n", $url); - $folderurl = KTBrowseUtil::getUrlForFolder($this->oFolder); - printf('

' . _kt('Once your download is complete, click here to return to the original folder') . "

\n", $folderurl); + print($sReturn); printf("\n"); printf('', $url); @@ -230,8 +252,7 @@ class KTBulkExportAction extends KTFolderAction { function do_downloadZipFile() { $sCode = $this->oValidator->validateString($_REQUEST['exportcode']); - $folderName = $this->oFolder->getName(); - $this->oZip = new ZipFolder($folderName); + $this->oZip = new ZipFolder('', $sCode); $res = $this->oZip->downloadZipFile($sCode); diff --git a/sql/mysql/install/data.sql b/sql/mysql/install/data.sql index dd20153..da64285 100644 --- a/sql/mysql/install/data.sql +++ b/sql/mysql/install/data.sql @@ -287,7 +287,9 @@ INSERT INTO `config_settings` VALUES (112, 'urls', 'Var Directory', 'The path to the var directory.', 'varDirectory', 'default', '${fileSystemRoot}/var', 'string', NULL, 1), (113, 'tweaks','Increment version on rename','Defines whether to update the version number if a document filename is changed/renamed.','incrementVersionOnRename','default','true','boolean',NULL,1), (114, 'ui', 'System URL', 'The system url, used in the main logo.', 'systemUrl', 'default', 'http://www.knowledgetree.com', 'string', '', 1), -(115, 'ldapAuthentication', 'Allow Moving Users in LDAP/AD', 'Moving users around within the LDAP or Active Directory structure will cause failed logins for these users. When this setting is enabled, a failed login will trigger a search for the user using their sAMAccountName setting and update their authentication details.', 'enableLdapUpdate', 'default', 'false', 'boolean', NULL, 1); +(115, 'ldapAuthentication', 'Allow Moving Users in LDAP/AD', 'Moving users around within the LDAP or Active Directory structure will cause failed logins for these users. When this setting is enabled, a failed login will trigger a search for the user using their sAMAccountName setting and update their authentication details.', 'enableLdapUpdate', 'default', 'false', 'boolean', NULL, 1), +(116, 'export', 'Use External Zip Binary', 'Utilises the external zip binary for compressing archives. The default is to use the PEAR archive class.', 'useBinary', 'default', 'false', 'boolean', NULL, 1), +(117, 'export', 'Use Bulk Download Queue', 'The bulk download can be large and can prevent normal browsing. The download queue performs the bulk downloads in the background.', 'useDownloadQueue', 'default', 'true', 'boolean', NULL, 1); /*!40000 ALTER TABLE `config_settings` ENABLE KEYS */; UNLOCK TABLES; @@ -1370,7 +1372,8 @@ INSERT INTO `scheduler_tasks` VALUES (7,'Cleanup Temporary Directory','search2/bin/cronCleanup.php','',0,'1min','2007-10-01 00:00:00',NULL,0,'enabled'), (8,'Disk Usage and Folder Utilisation Statistics','plugins/housekeeper/bin/UpdateStats.php','',0,'5mins','2007-10-01 00:00:00',NULL,0,'enabled'), (9,'Refresh Index Statistics','search2/bin/cronIndexStats.php','',0,'1min','2007-10-01',NULL,0,'enabled'), -(10,'Refresh Resource Dependancies','search2/bin/cronResources.php','',0,'1min','2007-10-01',NULL,0,'enabled'); +(10,'Refresh Resource Dependancies','search2/bin/cronResources.php','',0,'1min','2007-10-01',NULL,0,'enabled'), +(11,'Bulk Download Queue','lib/foldermanagement/downloadTask.php','',0,'1min','2007-10-01',NULL,0,'system'); /*!40000 ALTER TABLE `scheduler_tasks` ENABLE KEYS */; UNLOCK TABLES; diff --git a/sql/mysql/install/structure.sql b/sql/mysql/install/structure.sql index b6f5c88..80e1d99 100644 --- a/sql/mysql/install/structure.sql +++ b/sql/mysql/install/structure.sql @@ -639,6 +639,22 @@ CREATE TABLE `download_files` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- +-- Table structure for table `download_queue` +-- + +CREATE TABLE `download_queue` ( + `code` char(16) NOT NULL, + `folder_id` int(11) NOT NULL, + `object_id` int(11) NOT NULL, + `object_type` enum('document', 'folder') NOT NULL default 'folder', + `user_id` int(11) NOT NULL, + `date_added` timestamp NOT NULL default CURRENT_TIMESTAMP, + `status` tinyint(4) NOT NULL default 0, + `errors` mediumtext, + INDEX (`code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- -- Table structure for table `field_behaviour_options` -- diff --git a/sql/mysql/upgrade/3.6/download_queue.sql b/sql/mysql/upgrade/3.6/download_queue.sql new file mode 100644 index 0000000..6724583 --- /dev/null +++ b/sql/mysql/upgrade/3.6/download_queue.sql @@ -0,0 +1,23 @@ +-- +-- Table structure for table `download_queue` +-- + +CREATE TABLE `download_queue` ( + `code` char(16) NOT NULL, + `folder_id` int(11) NOT NULL, + `object_id` int(11) NOT NULL, + `object_type` enum('document', 'folder') NOT NULL default 'folder', + `user_id` int(11) NOT NULL, + `date_added` timestamp NOT NULL default CURRENT_TIMESTAMP, + `status` tinyint(4) NOT NULL default 0, + `errors` mediumtext, + INDEX (`code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT INTO `config_settings` (group_name, display_name, description, item, value, default_value, type, options, can_edit) +VALUES ('export', 'Use External Zip Binary', 'Utilises the external zip binary for compressing archives. The default is to use the PEAR archive class.', 'useBinary', 'default', 'false', 'boolean', NULL, 1), +('export', 'Use Bulk Download Queue', 'The bulk download can be large and can prevent normal browsing. The download queue performs the bulk downloads in +the background.', 'useDownloadQueue', 'default', 'true', 'boolean', NULL, 1); + +INSERT INTO `scheduler_tasks` (task, script_url, frequency, run_time, status) +VALUES ('Bulk Download Queue','lib/foldermanagement/downloadTask.php','1min','2007-10-01','system'); \ No newline at end of file diff --git a/templates/ktcore/action/bulk_download.smarty b/templates/ktcore/action/bulk_download.smarty new file mode 100644 index 0000000..e655936 --- /dev/null +++ b/templates/ktcore/action/bulk_download.smarty @@ -0,0 +1,47 @@ +

+The documents you have selected are being compressed and archived, once the archive has been created a download link will be displayed. +
Please do not close the page. +

+ + + + + + \ No newline at end of file -- libgit2 0.21.4