diff --git a/lib/foldermanagement/compressionArchiveUtil.inc.php b/lib/foldermanagement/compressionArchiveUtil.inc.php index d38aaa3..4dbba01 100644 --- a/lib/foldermanagement/compressionArchiveUtil.inc.php +++ b/lib/foldermanagement/compressionArchiveUtil.inc.php @@ -36,6 +36,8 @@ * */ +require_once('File/Archive.php'); + /** * Class to create and download a zip file */ @@ -50,6 +52,8 @@ class ZipFolder { var $aReplaceKeys = array(); var $aReplaceValues = array(); var $sOutputEncoding = 'UTF-8'; + var $extension = 'zip'; + var $exportCode = null; /** * Constructor @@ -57,22 +61,22 @@ class ZipFolder { * @param string $sZipFileName The name of the zip file - gets ignored at the moment. * @param string $exportCode The code to use if a zip file has already been created. */ - function ZipFolder($sZipFileName, $exportCode = null) { + function ZipFolder($sZipFileName = null, $exportCode = null, $extension = 'zip') { $this->oKTConfig =& KTConfig::getSingleton(); $this->oStorage =& KTStorageManagerUtil::getSingleton(); $this->sOutputEncoding = $this->oKTConfig->get('export/encoding', 'UTF-8'); + $this->extension = $extension; $this->sPattern = "[\*|\%|\\\|\/|\<|\>|\+|\:|\?|\||\'|\"]"; $this->sFolderPattern = "[\*|\%|\<|\>|\+|\:|\?|\||\'|\"]"; - // If the export code exists then a temp zip directory has already been created - if(!empty($exportCode)){ - $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode); + $this->exportCode = $exportCode; - if(!empty($aData)){ - $sTmpPath = $aData['dir']; - } + // Check if the temp directory has been created and stored in session + $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode); + if(!empty($aData) && isset($aData['dir'])){ + $sTmpPath = $aData['dir']; }else { $sBasedir = $this->oKTConfig->get("urls/tmpDirectory"); $sTmpPath = tempnam($sBasedir, 'kt_compress_zip'); @@ -100,6 +104,15 @@ class ZipFolder { $this->aReplaceValues = array_values($aReplace); } + static public function get($exportCode) + { + static $zipFolder = null; + if(is_null($zipFolder)){ + $zipFolder = new ZipFolder('', $exportCode); + } + return $zipFolder; + } + /** * Return the full path * @@ -154,7 +167,7 @@ class ZipFolder { $sOrigFile = $this->oStorage->temporaryFile($oDocument); $sFilename = $sParentFolder.'/'.$sDocName; - copy($sOrigFile, $sFilename); + @copy($sOrigFile, $sFilename); $this->aPaths[] = $sDocPath.'/'.$sDocName; return true; @@ -190,49 +203,62 @@ class ZipFolder { return PEAR::raiseError(_kt("No folders or documents found to compress")); } + $config = KTConfig::getSingleton(); + $useBinary = $config->get('export/useBinary', false); + // Set environment language to output character encoding $loc = $this->sOutputEncoding; putenv("LANG=$loc"); putenv("LANGUAGE=$loc"); $loc = setlocale(LC_ALL, $loc); + if($useBinary){ + $sManifest = sprintf("%s/%s", $this->sTmpPath, "MANIFEST"); + file_put_contents($sManifest, join("\n", $this->aPaths)); + } - - $sManifest = sprintf("%s/%s", $this->sTmpPath, "MANIFEST"); - file_put_contents($sManifest, join("\n", $this->aPaths)); - $sZipFile = sprintf("%s/%s.zip", $this->sTmpPath, $this->sZipFileName); + $sZipFile = sprintf("%s/%s.".$this->extension, $this->sTmpPath, $this->sZipFileName); $sZipFile = str_replace('<', '', str_replace('', '', str_replace('>', '', $sZipFile))); - $sZipCommand = KTUtil::findCommand("export/zip", "zip"); - $aCmd = array($sZipCommand, "-r", $sZipFile, ".", "-i@MANIFEST"); - $sOldPath = getcwd(); - chdir($this->sTmpPath); - - // Note that the popen means that pexec will return a file descriptor - $aOptions = array('popen' => 'r'); - $fh = KTUtil::pexec($aCmd, $aOptions); - - if($bEchoStatus){ - $last_beat = time(); - while(!feof($fh)) { - if ($i % 1000 == 0) { - $this_beat = time(); - if ($last_beat + 1 < $this_beat) { - $last_beat = $this_beat; - print " "; + + if($useBinary){ + $sZipCommand = KTUtil::findCommand("export/zip", "zip"); + $aCmd = array($sZipCommand, "-r", $sZipFile, ".", "-i@MANIFEST"); + $sOldPath = getcwd(); + chdir($this->sTmpPath); + + // Note that the popen means that pexec will return a file descriptor + $aOptions = array('popen' => 'r'); + $fh = KTUtil::pexec($aCmd, $aOptions); + + if($bEchoStatus){ + $last_beat = time(); + while(!feof($fh)) { + if ($i % 1000 == 0) { + $this_beat = time(); + if ($last_beat + 1 < $this_beat) { + $last_beat = $this_beat; + print " "; + } } + $contents = fread($fh, 4096); + if ($contents) { + print nl2br($this->_convertEncoding($contents, false)); + } + $i++; } - $contents = fread($fh, 4096); - if ($contents) { - print nl2br($this->_convertEncoding($contents, false)); - } - $i++; } + pclose($fh); + }else{ + // Create the zip archive using the PEAR File_Archive + File_Archive::extract( + File_Archive::read($this->sTmpPath.'/Root Folder'), + File_Archive::toArchive($this->sZipFileName.'.'.$this->extension, File_Archive::toFiles($this->sTmpPath), $this->extension) + ); } - pclose($fh); // Save the zip file and path into session $_SESSION['zipcompression'] = KTUtil::arrayGet($_SESSION, 'zipcompression', array()); - $sExportCode = KTUtil::randomString(); + $sExportCode = $this->exportCode; $_SESSION['zipcompression'][$sExportCode] = array( 'file' => $sZipFile, 'dir' => $this->sTmpPath, @@ -262,18 +288,41 @@ class ZipFolder { } if (!file_exists($sZipFile)) { - return PEAR::raiseError(_kt('The ZIP file can only be downloaded once - if you cancel the download, you will need to reload the page.')); + return PEAR::raiseError(_kt('The zip file has not been created, if you are downloading a large number of documents + or a large document then it may take a few seconds to finish. Try refreshing the page.')); } $mimeType = 'application/zip; charset=utf-8;'; $fileSize = filesize($sZipFile); - $fileName = $this->sZipFileName . '.zip'; + $fileName = $this->sZipFileName .'.'.$this->extension; KTUtil::download($sZipFile, $mimeType, $fileSize, $fileName); KTUtil::deleteDirectory($sTmpPath); return true; } + function checkArchiveExists($exportCode = null) + { + if(!(isset($exportCode) && !empty($exportCode))) { + $exportCode = KTUtil::arrayGet($_SESSION['zipcompression'], 'exportcode'); + } + + $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode); + + if(!empty($aData)){ + $sZipFile = $aData['file']; + $sTmpPath = $aData['dir']; + }else{ + $sZipFile = $this->sZipFile; + $sTmpPath = $this->sTmpPath; + } + + if (!file_exists($sZipFile)) { + return false; + } + return true; + } + /** * Check that iconv exists and that the selected encoding is supported. */ @@ -301,5 +350,407 @@ class ZipFolder { return iconv($this->sOutputEncoding, "UTF-8", $sMystring); } } + + static public function checkDownloadSize($object) + { + return true; + + if($object instanceof Document || $object instanceof DocumentProxy){ + } + + if($object instanceof Folder || $object instanceof FolderProxy){ + $id = $object->iId; + + // If we're working with the root folder + if($id = 1){ + $sql = 'SELECT count(*) as cnt FROM documents where folder_id = 1'; + }else{ + $sql[] = "SELECT count(*) as cnt FROM documents where parent_folder_ids like '%,?' OR parent_folder_ids like '%,?,%' OR folder_id = ?"; + $sql[] = array($id, $id, $id); + } + + /* + SELECT count(*) FROM documents d + INNER JOIN document_metadata_version m ON d.metadata_version_id = m.id + INNER JOIN document_content_version c ON m.content_version_id = c.id + where (d.parent_folder_ids like '%,12' OR d.parent_folder_ids like '%,12,%' OR d.folder_id = 12) AND d.status_id < 3 AND size > 100000 + */ + + $result = DBUtil::getOneResult($sql); + + if($result['cnt'] > 10){ + return true; + } + } + + return false; + } +} + +/** + * Class to manage the queue of bulk downloads + * + */ +class DownloadQueue +{ + private $bNoisy; + private $bNotifications; + private $errors; + + public function __construct() + { + $config = KTConfig::getSingleton(); + $this->bNoisy = $config->get('tweaks/noisyBulkOperations', false); + $this->bNotifications = ($config->get('export/enablenotifications', 'on') == 'on') ? true : false; + } + + /** + * Add an item to the download queue + * + * @param string $code The identification string for the download + * @param string $id The object id + * @param string $type The type of object Folder | Document + */ + static public function addItem($code, $folderId, $id, $type) + { + $fields = array(); + $fields['code'] = $code; + $fields['folder_id'] = $folderId; + $fields['object_id'] = $id; + $fields['object_type'] = $type; + $fields['user_id'] = $_SESSION['userID']; + $fields['date_added'] = date('Y-m-d H:i:s'); + + $res = DBUtil::autoInsert('download_queue', $fields); + } + + public function removeItem($code) + { + $where = array('code' => $code); + $res = DBUtil::whereDelete('download_queue', $where); + return $res; + } + + public function getQueue() + { + $sql = 'SELECT * FROM download_queue d ORDER BY date_added, code'; + $rows = DBUtil::getResultArray($sql); + + if(PEAR::isError($rows)){ + return $rows; + } + + $queue = array(); + foreach ($rows as $item){ + $queue[$item['code']][] = $item; + } + return $queue; + } + + public function getItemStatus($code) + { + $sql = array(); + $sql[] = 'SELECT status, errors FROM download_queue WHERE code = ?'; + $sql[] = $code; + $result = DBUtil::getResultArray($sql); + return $result; + } + + public function setItemStatus($code, $status = 1, $error = null) + { + $fields = array(); + $fields['status'] = $status; + $fields['errors'] = !empty($error) ? json_encode($error) : null; + $where = array('code' => $code); + $res = DBUtil::whereUpdate('download_queue', $fields, $where); + return $res; + } + + public function processQueue() + { + global $default; + + // get items from queue + $queue = $this->getQueue(); + if(PEAR::isError($queue)){ + $default->log->debug('Download Queue: error on fetching queue - '.$queue->getMessage()); + return false; + } + + // Loop through items and create downloads + foreach ($queue as $code => $download){ + // reset the error messages + $this->errors = null; + + // if the user_id is not set then skip + if(!isset($download[0]['user_id']) || empty($download[0]['user_id'])){ + $default->log->debug('Download Queue: no user id set for download code '.$code); + $error = array(_kt('No user id has been set, the archive cannot be created.')); + $result = $this->setItemStatus($code, 2, $error); + continue; + } + + // Force a session for the user + $_SESSION['userID'] = $download[0]['user_id']; + $baseFolderId = $download[0]['folder_id']; + + // Create a new instance of the archival class + $zip = new ZipFolder('', $code); + $res = $zip->checkConvertEncoding(); + + if(PEAR::isError($res)){ + $default->log->error('Download Queue: Archive class check convert encoding error - '.$res->getMessage()); + $error = array(_kt('The archive cannot be created. An error occurred in the encoding.')); + $result = $this->setItemStatus($code, 2, $error); + continue; + } + + $default->log->debug('Download Queue: Creating download for user: '.$_SESSION['userID'].', code: '.$code); + + DBUtil::startTransaction(); + + // Add the individual files and folders into the archive + foreach ($download as $item){ + if($item['object_type'] == 'document'){ + $docId = $item['object_id']; + $this->addDocument($zip, $docId); + } + if($item['object_type'] == 'folder'){ + $folderId = $item['object_id']; + $this->addFolder($zip, $folderId); + } + } + + $res = $zip->createZipFile(); + + if(PEAR::isError($res)){ + $default->log->debug('Download Queue: Archive could not be created. Exiting transaction. '.$res->getMessage()); + DBUtil::rollback(); + + $error = array(_kt('The archive could not be created.')); + $result = $this->setItemStatus($code, 2, $error); + continue; + } + + $default->log->debug('Download Queue: Archival successful'); + + $oTransaction = KTFolderTransaction::createFromArray(array( + 'folderid' => $baseFolderId, + 'comment' => "Bulk export", + 'transactionNS' => 'ktstandard.transactions.bulk_export', + 'userid' => $_SESSION['userID'], + 'ip' => Session::getClientIP(), + )); + + if(PEAR::isError($oTransaction)){ + $default->log->debug('Download Queue: transaction could not be logged. '.$oTransaction->getMessage()); + } + + DBUtil::commit(); + + // Set status for the download + $this->errors['archive'] = $_SESSION['zipcompression']; + $result = $this->setItemStatus($code, 1, $this->errors); + if(PEAR::isError($result)){ + $default->log->error('Download Queue: item status could not be set for user: '.$_SESSION['userID'].', code: '.$code.', error: '.$result->getMessage()); + } + // reset the error messages + $this->errors = null; + $_SESSION['zipcompression'] = null; + } + } + + public function addDocument(&$zip, $docId) + { + + $oDocument = Document::get($docId); + if(PEAR::isError($oDocument)){ + $this->errors[] = _kt('Document cannot be exported, an error occurred: ').$oDocument->getMessage(); + return $oDocument; + } + + if ($this->bNoisy) { + $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array()); + $oDocumentTransaction->create(); + } + + // fire subscription alerts for the downloaded document - if global config is set + if($this->bNotifications){ + $oSubscriptionEvent = new SubscriptionEvent(); + $oFolder = Folder::get($oDocument->getFolderID()); + $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder); + } + + return $zip->addDocumentToZip($oDocument); + } + + public function addFolder(&$zip, $folderId) + { + $oFolder = Folder::get($folderId); + + if(PEAR::isError($oFolder)){ + $this->errors[] = _kt('Folder cannot be exported, an error occurred: ').$oFolder->getMessage(); + return $oFolder; + } + + $sFolderDocs = $oFolder->getDocumentIDs($folderId); + if(PEAR::isError($sFolderDocs)){ + $default->log->error('Download Queue: get document ids for folder caused an error: '.$sFolderDocs->getMessage()); + $sFolderDocs = ''; + } + + // Add folder to zip + $zip->addFolderToZip($oFolder); + + $aDocuments = array(); + if(!empty($sFolderDocs)){ + $aDocuments = explode(',', $sFolderDocs); + } + + // Get all the folders within the current folder + $sWhereClause = "parent_folder_ids like '%,{$folderId}' + OR parent_folder_ids like '%,{$folderId},%' + OR parent_id = {$folderId}"; + + $aFolderList = $oFolder->getList($sWhereClause); + $aLinkingFolders = $this->getLinkingEntities($aFolderList); + $aFolderList = array_merge($aFolderList,$aLinkingFolders); + + $aFolderObjects = array(); + $aFolderObjects[$folderId] = $oFolder; + + // Export the folder structure to ensure the export of empty directories + if(!empty($aFolderList)){ + foreach($aFolderList as $k => $oFolderItem){ + if($oFolderItem->isSymbolicLink()){ + $oFolderItem = $oFolderItem->getLinkedFolder(); + } + if(Permission::userHasFolderReadPermission($oFolderItem)){ + // Get documents for each folder + $sFolderItemId = $oFolderItem->getID(); + $sFolderItemDocs = $oFolderItem->getDocumentIDs($sFolderItemId); + + if(!empty($sFolderItemDocs)){ + $aFolderDocs = explode(',', $sFolderItemDocs); + $aDocuments = array_merge($aDocuments, $aFolderDocs); + } + $zip->addFolderToZip($oFolderItem); + $aFolderObjects[$oFolderItem->getId()] = $oFolderItem; + } + } + } + + // Add all documents to the export + if(!empty($aDocuments)){ + foreach($aDocuments as $sDocumentId){ + $oDocument = Document::get($sDocumentId); + if($oDocument->isSymbolicLink()){ + $oDocument->switchToLinkedCore(); + } + if(Permission::userHasDocumentReadPermission($oDocument)){ + + if(!KTWorkflowUtil::actionEnabledForDocument($oDocument, 'ktcore.actions.document.view')){ + $this->errors[] = $oDocument->getName().': '._kt('Document cannot be exported as it is restricted by the workflow.'); + continue; + } + + $sDocFolderId = $oDocument->getFolderID(); + $oFolder = isset($aFolderObjects[$sDocFolderId]) ? $aFolderObjects[$sDocFolderId] : Folder::get($sDocFolderId); + + if ($this->bNoisy) { + $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array()); + $oDocumentTransaction->create(); + } + + // fire subscription alerts for the downloaded document + if($this->bNotifications){ + $oSubscriptionEvent = new SubscriptionEvent(); + $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder); + } + + $zip->addDocumentToZip($oDocument, $oFolder); + } + } + } + } + + function getLinkingEntities($aFolderList){ + $aSearchFolders = array(); + if(!empty($aFolderList)){ + foreach($aFolderList as $oFolderItem){ + if(Permission::userHasFolderReadPermission($oFolderItem)){ + // If it is a shortcut, we should do some more searching + if($oFolderItem->isSymbolicLink()){ + $oFolderItem = $oFolderItem->getLinkedFolder(); + $aSearchFolders[] = $oFolderItem->getID(); + } + } + } + } + $aLinkingFolders = array(); + $aSearchCompletedFolders = array(); + $count = 0; + while(count($aSearchFolders)>0){ + $count++; + $oFolder = Folder::get(array_pop($aSearchFolders)); + $folderId = $oFolder->getId(); + // Get all the folders within the current folder + $sWhereClause = "parent_folder_ids = '{$folderId}' OR + parent_folder_ids LIKE '{$folderId},%' OR + parent_folder_ids LIKE '%,{$folderId},%' OR + parent_folder_ids LIKE '%,{$folderId}'"; + $aFolderList = $this->oFolder->getList($sWhereClause); + foreach($aFolderList as $oFolderItem){ + if($oFolderItem->isSymbolicLink()){ + $oFolderItem = $oFolderItem->getLinkedFolder(); + } + if(Permission::userHasFolderReadPermission($oFolderItem)){ + if($aSearchCompletedFolders[$oFolderItem->getID()] != true){ + $aSearchFolders[] = $oFolderItem->getID(); + $aSearchCompletedFolders[$oFolderItem->getID()] = true; + } + } + } + if(!isset($aLinkingFolders[$oFolder->getId()])){ + $aLinkingFolders[$oFolder->getId()] = $oFolder; + } + } + return $aLinkingFolders; + } + + public function isDownloadAvailable($code) + { + $check = $this->getItemStatus($code); + $status = $check[0]['status']; + + if($status < 1){ + return false; + } + + $message = $check[0]['errors']; + $message = json_decode($message, true); + + if($status > 1){ + return $message; + } + + // Create the archive session variables + $_SESSION['zipcompression'] = $message['archive']; + unset($message['archive']); + + // Check that the archive has been created + $zip = new ZipFolder('', $code); + if($zip->checkArchiveExists($code)){ + // Clean up the download queue and return errors + $this->removeItem($code); + return $message; + } + return false; + } + + public function isLocked() + { + return false; + } } ?> diff --git a/lib/foldermanagement/downloadTask.php b/lib/foldermanagement/downloadTask.php new file mode 100644 index 0000000..cd33cbb --- /dev/null +++ b/lib/foldermanagement/downloadTask.php @@ -0,0 +1,47 @@ +isDownloadAvailable($code); + + if($status === false){ + echo 'wait'; + }else{ + $str = ''; + // display any error messages + if(!empty($status)){ + $str = '
| '.$msg.' |
' . _kt('Return to the original folder') . "
\n", $folderurl); + $download_url = KTUtil::addQueryStringSelf("action=downloadZipFile&fFolderId={$this->oFolder->getId()}&exportcode={$this->sExportCode}"); + + if($useQueue){ + $result = parent::do_performaction(); + + $url = KTUtil::kt_url() . '/lib/foldermanagement/downloadTask.php'; + + $oTemplating =& KTTemplating::getSingleton(); + $oTemplate = $oTemplating->loadTemplate('ktcore/action/bulk_download'); + + $aParams = array( + 'return' => $sReturn, + 'url' => $url, + 'code' => $this->sExportCode, + 'download_url' => $download_url + ); + return $oTemplate->render($aParams); + } + + $this->oZip = new ZipFolder('', $this->sExportCode); + $res = $this->oZip->checkConvertEncoding(); if(PEAR::isError($res)){ $this->addErrorMessage($res->getMessage()); @@ -1087,16 +1111,14 @@ class KTBrowseBulkExportAction extends KTBulkAction { } $this->startTransaction(); - $oKTConfig =& KTConfig::getSingleton(); - $this->bNoisy = $oKTConfig->get("tweaks/noisyBulkOperations"); - - $this->bNotifications = ($oKTConfig->get('export/enablenotifications', 'on') == 'on') ? true : false; $result = parent::do_performaction(); + $sExportCode = $this->oZip->createZipFile(); if(PEAR::isError($sExportCode)){ $this->addErrorMessage($sExportCode->getMessage()); + $this->rollbackTransaction(); return $sReturn; } @@ -1110,23 +1132,28 @@ class KTBrowseBulkExportAction extends KTBulkAction { $this->commitTransaction(); - $url = KTUtil::addQueryStringSelf(sprintf('action=downloadZipFile&fFolderId=%d&exportcode=%s', $this->oFolder->getId(), $sExportCode)); - $str = sprintf('' . _kt('Your download will begin shortly. If you are not automatically redirected to your download, please click here ') . "
\n", $url); + $str = sprintf('' . _kt('Your download will begin shortly. If you are not automatically redirected to your download, please click here ') . "
\n", $download_url); $str .= sprintf('' . _kt('Once your download is complete, click here to return to the original folder') . "
\n", $folderurl); - //$str .= sprintf("