KTS-4140. Moved the bulk download into a background task. Changed it to use PEAR…

… archive functionality instead of the zip binary. "CLONE -error: The ZIP file can only be downloaded once - if you cancel the download, you will need to reload the page. (SUP-1680)" In progress. Committed by: Megan Watson Reviewed by: Kevin Fourie

KTS-4140. Moved the bulk download into a background task. Changed it to use PEAR…
… archive functionality instead of the zip binary. "CLONE -error: The ZIP file can only be downloaded once - if you cancel the download, you will need to reload the page. (SUP-1680)" In progress. Committed by: Megan Watson Reviewed by: Kevin Fourie
Megan
1 parent 48c32a49
Showing 8 changed files with 715 additions and 157 deletions
lib/foldermanagement/compressionArchiveUtil.inc.php
lib/foldermanagement/downloadTask.php
plugins/ktcore/KTBulkActions.php
plugins/ktstandard/KTBulkExportPlugin.php
sql/mysql/install/data.sql
sql/mysql/install/structure.sql
sql/mysql/upgrade/3.6/download_queue.sql
templates/ktcore/action/bulk_download.smarty
@@ -36,6 +36,8 @@
  *
  */
+require_once('File/Archive.php');
+
 /**
 * Class to create and download a zip file
 */
@@ -50,6 +52,8 @@ class ZipFolder {
     var $aReplaceKeys = array();
     var $aReplaceValues = array();
     var $sOutputEncoding = 'UTF-8';
+    var $extension = 'zip';
+    var $exportCode = null;
     /**
     * Constructor
@@ -57,22 +61,22 @@ class ZipFolder {
     * @param string $sZipFileName The name of the zip file - gets ignored at the moment.
     * @param string $exportCode The code to use if a zip file has already been created.
     */
-    function ZipFolder($sZipFileName, $exportCode = null) {
+    function ZipFolder($sZipFileName = null, $exportCode = null, $extension = 'zip') {
         $this->oKTConfig =& KTConfig::getSingleton();
         $this->oStorage =& KTStorageManagerUtil::getSingleton();
         $this->sOutputEncoding = $this->oKTConfig->get('export/encoding', 'UTF-8');
+        $this->extension = $extension;
         $this->sPattern = "[\*|\%|\\\|\/|\<|\>|\+|\:|\?|\||\'|\"]";
         $this->sFolderPattern = "[\*|\%|\<|\>|\+|\:|\?|\||\'|\"]";
-        // If the export code exists then a temp zip directory has already been created
-        if(!empty($exportCode)){
-            $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode);
+        $this->exportCode = $exportCode;
-            if(!empty($aData)){
-                $sTmpPath = $aData['dir'];
-            }
+        // Check if the temp directory has been created and stored in session
+        $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode);
+        if(!empty($aData) && isset($aData['dir'])){
+            $sTmpPath = $aData['dir'];
         }else {
             $sBasedir = $this->oKTConfig->get("urls/tmpDirectory");
             $sTmpPath = tempnam($sBasedir, 'kt_compress_zip');
@@ -100,6 +104,15 @@ class ZipFolder {
         $this->aReplaceValues = array_values($aReplace);
     }
+    static public function get($exportCode)
+    {
+        static $zipFolder = null;
+        if(is_null($zipFolder)){
+            $zipFolder = new ZipFolder('', $exportCode);
+        }
+        return $zipFolder;
+    }
+
     /**
      * Return the full path
      *
@@ -154,7 +167,7 @@ class ZipFolder {
         $sOrigFile = $this->oStorage->temporaryFile($oDocument);
         $sFilename = $sParentFolder.'/'.$sDocName;
-        copy($sOrigFile, $sFilename);
+        @copy($sOrigFile, $sFilename);
         $this->aPaths[] = $sDocPath.'/'.$sDocName;
         return true;
@@ -190,49 +203,62 @@ class ZipFolder {
             return PEAR::raiseError(_kt("No folders or documents found to compress"));
         }
+        $config = KTConfig::getSingleton();
+        $useBinary = $config->get('export/useBinary', false);
+
         // Set environment language to output character encoding
         $loc = $this->sOutputEncoding;
         putenv("LANG=$loc");
         putenv("LANGUAGE=$loc");
         $loc = setlocale(LC_ALL, $loc);
+        if($useBinary){
+            $sManifest = sprintf("%s/%s", $this->sTmpPath, "MANIFEST");
+            file_put_contents($sManifest, join("\n", $this->aPaths));
+        }
-
-        $sManifest = sprintf("%s/%s", $this->sTmpPath, "MANIFEST");
-        file_put_contents($sManifest, join("\n", $this->aPaths));
-        $sZipFile = sprintf("%s/%s.zip", $this->sTmpPath, $this->sZipFileName);
+        $sZipFile = sprintf("%s/%s.".$this->extension, $this->sTmpPath, $this->sZipFileName);
         $sZipFile = str_replace('<', '', str_replace('</', '', str_replace('>', '', $sZipFile)));
-        $sZipCommand = KTUtil::findCommand("export/zip", "zip");
-        $aCmd = array($sZipCommand, "-r", $sZipFile, ".", "-i@MANIFEST");
-        $sOldPath = getcwd();
-        chdir($this->sTmpPath);
-
-        // Note that the popen means that pexec will return a file descriptor
-        $aOptions = array('popen' => 'r');
-        $fh = KTUtil::pexec($aCmd, $aOptions);
-
-        if($bEchoStatus){
-            $last_beat = time();
-            while(!feof($fh)) {
-                if ($i % 1000 == 0) {
-                    $this_beat = time();
-                    if ($last_beat + 1 < $this_beat) {
-                        $last_beat = $this_beat;
-                        print "&nbsp;";
+
+        if($useBinary){
+            $sZipCommand = KTUtil::findCommand("export/zip", "zip");
+            $aCmd = array($sZipCommand, "-r", $sZipFile, ".", "-i@MANIFEST");
+            $sOldPath = getcwd();
+            chdir($this->sTmpPath);
+
+            // Note that the popen means that pexec will return a file descriptor
+            $aOptions = array('popen' => 'r');
+            $fh = KTUtil::pexec($aCmd, $aOptions);
+
+            if($bEchoStatus){
+                $last_beat = time();
+                while(!feof($fh)) {
+                    if ($i % 1000 == 0) {
+                        $this_beat = time();
+                        if ($last_beat + 1 < $this_beat) {
+                            $last_beat = $this_beat;
+                            print "&nbsp;";
+                        }
                     }
+                    $contents = fread($fh, 4096);
+                    if ($contents) {
+                        print nl2br($this->_convertEncoding($contents, false));
+                    }
+                    $i++;
                 }
-                $contents = fread($fh, 4096);
-                if ($contents) {
-                    print nl2br($this->_convertEncoding($contents, false));
-                }
-                $i++;
             }
+            pclose($fh);
+        }else{
+            // Create the zip archive using the PEAR File_Archive
+            File_Archive::extract(
+                File_Archive::read($this->sTmpPath.'/Root Folder'),
+                File_Archive::toArchive($this->sZipFileName.'.'.$this->extension, File_Archive::toFiles($this->sTmpPath), $this->extension)
+            );
         }
-        pclose($fh);
         // Save the zip file and path into session
         $_SESSION['zipcompression'] = KTUtil::arrayGet($_SESSION, 'zipcompression', array());
-        $sExportCode = KTUtil::randomString();
+        $sExportCode = $this->exportCode;
         $_SESSION['zipcompression'][$sExportCode] = array(
             'file' => $sZipFile,
             'dir' => $this->sTmpPath,
@@ -262,18 +288,41 @@ class ZipFolder {
         }
         if (!file_exists($sZipFile)) {
-            return PEAR::raiseError(_kt('The ZIP file can only be downloaded once - if you cancel the download, you will need to reload the page.'));
+            return PEAR::raiseError(_kt('The zip file has not been created, if you are downloading a large number of documents
+            or a large document then it may take a few seconds to finish. Try refreshing the page.'));
         }
         $mimeType = 'application/zip; charset=utf-8;';
         $fileSize = filesize($sZipFile);
-        $fileName = $this->sZipFileName . '.zip';
+        $fileName = $this->sZipFileName .'.'.$this->extension;
         KTUtil::download($sZipFile, $mimeType, $fileSize, $fileName);
         KTUtil::deleteDirectory($sTmpPath);
         return true;
     }
+    function checkArchiveExists($exportCode = null)
+    {
+        if(!(isset($exportCode) && !empty($exportCode))) {
+            $exportCode = KTUtil::arrayGet($_SESSION['zipcompression'], 'exportcode');
+        }
+
+        $aData = KTUtil::arrayGet($_SESSION['zipcompression'], $exportCode);
+
+        if(!empty($aData)){
+            $sZipFile = $aData['file'];
+            $sTmpPath = $aData['dir'];
+        }else{
+            $sZipFile = $this->sZipFile;
+            $sTmpPath = $this->sTmpPath;
+        }
+
+        if (!file_exists($sZipFile)) {
+            return false;
+        }
+        return true;
+    }
+
     /**
     * Check that iconv exists and that the selected encoding is supported.
     */
@@ -301,5 +350,407 @@ class ZipFolder {
     		return iconv($this->sOutputEncoding, "UTF-8", $sMystring);
     	}
     }
+
+    static public function checkDownloadSize($object)
+    {
+        return true;
+
+        if($object instanceof Document || $object instanceof DocumentProxy){
+        }
+
+        if($object instanceof Folder || $object instanceof FolderProxy){
+            $id = $object->iId;
+
+            // If we're working with the root folder
+            if($id = 1){
+                $sql = 'SELECT count(*) as cnt FROM documents where folder_id = 1';
+            }else{
+                $sql[] = "SELECT count(*) as cnt FROM documents where parent_folder_ids like '%,?' OR parent_folder_ids like '%,?,%' OR folder_id = ?";
+                $sql[] = array($id, $id, $id);
+            }
+
+            /*
+            SELECT count(*) FROM documents d
+            INNER JOIN document_metadata_version m ON d.metadata_version_id = m.id
+            INNER JOIN document_content_version c ON m.content_version_id = c.id
+            where (d.parent_folder_ids like '%,12' OR d.parent_folder_ids like '%,12,%' OR d.folder_id = 12) AND d.status_id < 3 AND size > 100000
+            */
+
+            $result = DBUtil::getOneResult($sql);
+
+            if($result['cnt'] > 10){
+                return true;
+            }
+        }
+
+        return false;
+    }
+}
+
+/**
+ * Class to manage the queue of bulk downloads
+ *
+ */
+class DownloadQueue
+{
+    private $bNoisy;
+    private $bNotifications;
+    private $errors;
+
+    public function __construct()
+    {
+        $config = KTConfig::getSingleton();
+        $this->bNoisy = $config->get('tweaks/noisyBulkOperations', false);
+        $this->bNotifications = ($config->get('export/enablenotifications', 'on') == 'on') ? true : false;
+    }
+
+    /**
+     * Add an item to the download queue
+     *
+     * @param string $code The identification string for the download
+     * @param string $id The object id
+     * @param string $type The type of object Folder | Document
+     */
+    static public function addItem($code, $folderId, $id, $type)
+    {
+        $fields = array();
+        $fields['code'] = $code;
+        $fields['folder_id'] = $folderId;
+        $fields['object_id'] = $id;
+        $fields['object_type'] = $type;
+        $fields['user_id'] = $_SESSION['userID'];
+        $fields['date_added'] = date('Y-m-d H:i:s');
+
+        $res = DBUtil::autoInsert('download_queue', $fields);
+    }
+
+    public function removeItem($code)
+    {
+        $where = array('code' => $code);
+        $res = DBUtil::whereDelete('download_queue', $where);
+        return $res;
+    }
+
+    public function getQueue()
+    {
+        $sql = 'SELECT * FROM download_queue d ORDER BY date_added, code';
+        $rows = DBUtil::getResultArray($sql);
+
+        if(PEAR::isError($rows)){
+            return $rows;
+        }
+
+        $queue = array();
+        foreach ($rows as $item){
+            $queue[$item['code']][] = $item;
+        }
+        return $queue;
+    }
+
+    public function getItemStatus($code)
+    {
+        $sql = array();
+        $sql[] = 'SELECT status, errors FROM download_queue WHERE code = ?';
+        $sql[] = $code;
+        $result = DBUtil::getResultArray($sql);
+        return $result;
+    }
+
+    public function setItemStatus($code, $status = 1, $error = null)
+    {
+        $fields = array();
+        $fields['status'] = $status;
+        $fields['errors'] = !empty($error) ? json_encode($error) : null;
+        $where = array('code' => $code);
+        $res = DBUtil::whereUpdate('download_queue', $fields, $where);
+        return $res;
+    }
+
+    public function processQueue()
+    {
+        global $default;
+
+        // get items from queue
+        $queue = $this->getQueue();
+        if(PEAR::isError($queue)){
+            $default->log->debug('Download Queue: error on fetching queue - '.$queue->getMessage());
+            return false;
+        }
+
+        // Loop through items and create downloads
+        foreach ($queue as $code => $download){
+            // reset the error messages
+            $this->errors = null;
+
+            // if the user_id is not set then skip
+            if(!isset($download[0]['user_id']) || empty($download[0]['user_id'])){
+                $default->log->debug('Download Queue: no user id set for download code '.$code);
+                $error = array(_kt('No user id has been set, the archive cannot be created.'));
+                $result = $this->setItemStatus($code, 2, $error);
+                continue;
+            }
+
+            // Force a session for the user
+            $_SESSION['userID'] = $download[0]['user_id'];
+            $baseFolderId = $download[0]['folder_id'];
+
+            // Create a new instance of the archival class
+            $zip = new ZipFolder('', $code);
+            $res = $zip->checkConvertEncoding();
+
+            if(PEAR::isError($res)){
+                $default->log->error('Download Queue: Archive class check convert encoding error - '.$res->getMessage());
+                $error = array(_kt('The archive cannot be created. An error occurred in the encoding.'));
+                $result = $this->setItemStatus($code, 2, $error);
+                continue;
+            }
+
+            $default->log->debug('Download Queue: Creating download for user: '.$_SESSION['userID'].', code: '.$code);
+
+            DBUtil::startTransaction();
+
+            // Add the individual files and folders into the archive
+            foreach ($download as $item){
+                if($item['object_type'] == 'document'){
+                    $docId = $item['object_id'];
+                    $this->addDocument($zip, $docId);
+                }
+                if($item['object_type'] == 'folder'){
+                    $folderId = $item['object_id'];
+                    $this->addFolder($zip, $folderId);
+                }
+            }
+
+            $res = $zip->createZipFile();
+
+            if(PEAR::isError($res)){
+                $default->log->debug('Download Queue: Archive could not be created. Exiting transaction. '.$res->getMessage());
+                DBUtil::rollback();
+
+                $error = array(_kt('The archive could not be created.'));
+                $result = $this->setItemStatus($code, 2, $error);
+                continue;
+            }
+
+            $default->log->debug('Download Queue: Archival successful');
+
+            $oTransaction = KTFolderTransaction::createFromArray(array(
+                'folderid' => $baseFolderId,
+                'comment' => "Bulk export",
+                'transactionNS' => 'ktstandard.transactions.bulk_export',
+                'userid' => $_SESSION['userID'],
+                'ip' => Session::getClientIP(),
+            ));
+
+            if(PEAR::isError($oTransaction)){
+                $default->log->debug('Download Queue: transaction could not be logged. '.$oTransaction->getMessage());
+            }
+
+            DBUtil::commit();
+
+            // Set status for the download
+            $this->errors['archive'] = $_SESSION['zipcompression'];
+            $result = $this->setItemStatus($code, 1, $this->errors);
+            if(PEAR::isError($result)){
+                $default->log->error('Download Queue: item status could not be set for user: '.$_SESSION['userID'].', code: '.$code.', error: '.$result->getMessage());
+            }
+            // reset the error messages
+            $this->errors = null;
+            $_SESSION['zipcompression'] = null;
+        }
+    }
+
+    public function addDocument(&$zip, $docId)
+    {
+
+        $oDocument = Document::get($docId);
+        if(PEAR::isError($oDocument)){
+            $this->errors[] = _kt('Document cannot be exported, an error occurred: ').$oDocument->getMessage();
+            return $oDocument;
+        }
+
+        if ($this->bNoisy) {
+            $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array());
+            $oDocumentTransaction->create();
+        }
+
+        // fire subscription alerts for the downloaded document - if global config is set
+        if($this->bNotifications){
+            $oSubscriptionEvent = new SubscriptionEvent();
+            $oFolder = Folder::get($oDocument->getFolderID());
+            $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder);
+        }
+
+        return $zip->addDocumentToZip($oDocument);
+    }
+
+    public function addFolder(&$zip, $folderId)
+    {
+        $oFolder = Folder::get($folderId);
+
+        if(PEAR::isError($oFolder)){
+            $this->errors[] = _kt('Folder cannot be exported, an error occurred: ').$oFolder->getMessage();
+            return $oFolder;
+        }
+
+        $sFolderDocs = $oFolder->getDocumentIDs($folderId);
+        if(PEAR::isError($sFolderDocs)){
+            $default->log->error('Download Queue: get document ids for folder caused an error: '.$sFolderDocs->getMessage());
+            $sFolderDocs = '';
+        }
+
+        // Add folder to zip
+        $zip->addFolderToZip($oFolder);
+
+        $aDocuments = array();
+        if(!empty($sFolderDocs)){
+            $aDocuments = explode(',', $sFolderDocs);
+        }
+
+        // Get all the folders within the current folder
+        $sWhereClause = "parent_folder_ids like '%,{$folderId}'
+            OR parent_folder_ids like '%,{$folderId},%'
+            OR parent_id = {$folderId}";
+
+        $aFolderList = $oFolder->getList($sWhereClause);
+		$aLinkingFolders = $this->getLinkingEntities($aFolderList);
+        $aFolderList = array_merge($aFolderList,$aLinkingFolders);
+
+        $aFolderObjects = array();
+        $aFolderObjects[$folderId] = $oFolder;
+
+        // Export the folder structure to ensure the export of empty directories
+        if(!empty($aFolderList)){
+            foreach($aFolderList as $k => $oFolderItem){
+                if($oFolderItem->isSymbolicLink()){
+                	$oFolderItem = $oFolderItem->getLinkedFolder();
+                }
+            	if(Permission::userHasFolderReadPermission($oFolderItem)){
+                    // Get documents for each folder
+                    $sFolderItemId = $oFolderItem->getID();
+                    $sFolderItemDocs = $oFolderItem->getDocumentIDs($sFolderItemId);
+
+                    if(!empty($sFolderItemDocs)){
+                        $aFolderDocs = explode(',', $sFolderItemDocs);
+                        $aDocuments = array_merge($aDocuments, $aFolderDocs);
+                    }
+                    $zip->addFolderToZip($oFolderItem);
+                    $aFolderObjects[$oFolderItem->getId()] = $oFolderItem;
+            	}
+            }
+        }
+
+        // Add all documents to the export
+        if(!empty($aDocuments)){
+            foreach($aDocuments as $sDocumentId){
+                $oDocument = Document::get($sDocumentId);
+             	if($oDocument->isSymbolicLink()){
+    				$oDocument->switchToLinkedCore();
+    			}
+    			if(Permission::userHasDocumentReadPermission($oDocument)){
+
+                    if(!KTWorkflowUtil::actionEnabledForDocument($oDocument, 'ktcore.actions.document.view')){
+                        $this->errors[] = $oDocument->getName().': '._kt('Document cannot be exported as it is restricted by the workflow.');
+                        continue;
+                    }
+
+                    $sDocFolderId = $oDocument->getFolderID();
+                    $oFolder = isset($aFolderObjects[$sDocFolderId]) ? $aFolderObjects[$sDocFolderId] : Folder::get($sDocFolderId);
+
+                    if ($this->bNoisy) {
+                        $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array());
+                        $oDocumentTransaction->create();
+                    }
+
+                    // fire subscription alerts for the downloaded document
+                    if($this->bNotifications){
+                        $oSubscriptionEvent = new SubscriptionEvent();
+                        $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder);
+                    }
+
+                    $zip->addDocumentToZip($oDocument, $oFolder);
+    			}
+            }
+        }
+    }
+
+    function getLinkingEntities($aFolderList){
+    	$aSearchFolders = array();
+    	if(!empty($aFolderList)){
+            foreach($aFolderList as $oFolderItem){
+            	if(Permission::userHasFolderReadPermission($oFolderItem)){
+	                // If it is a shortcut, we should do some more searching
+	                if($oFolderItem->isSymbolicLink()){
+	                    $oFolderItem = $oFolderItem->getLinkedFolder();
+	                    $aSearchFolders[] = $oFolderItem->getID();
+	                }
+            	}
+             }
+        }
+    	$aLinkingFolders = array();
+    	$aSearchCompletedFolders = array();
+    	$count = 0;
+        while(count($aSearchFolders)>0){
+        	$count++;
+        	$oFolder = Folder::get(array_pop($aSearchFolders));
+        	$folderId = $oFolder->getId();
+        	 // Get all the folders within the current folder
+            $sWhereClause = "parent_folder_ids = '{$folderId}' OR
+            parent_folder_ids LIKE '{$folderId},%' OR
+            parent_folder_ids LIKE '%,{$folderId},%' OR
+            parent_folder_ids LIKE '%,{$folderId}'";
+            $aFolderList = $this->oFolder->getList($sWhereClause);
+            foreach($aFolderList as $oFolderItem){
+	            if($oFolderItem->isSymbolicLink()){
+	            	$oFolderItem = $oFolderItem->getLinkedFolder();
+	            }
+				if(Permission::userHasFolderReadPermission($oFolderItem)){
+		            if($aSearchCompletedFolders[$oFolderItem->getID()] != true){
+	            		$aSearchFolders[] = $oFolderItem->getID();
+	            		$aSearchCompletedFolders[$oFolderItem->getID()] = true;
+	            	}
+				}
+            }
+            if(!isset($aLinkingFolders[$oFolder->getId()])){
+            	$aLinkingFolders[$oFolder->getId()] = $oFolder;
+            }
+        }
+        return $aLinkingFolders;
+    }
+
+    public function isDownloadAvailable($code)
+    {
+        $check = $this->getItemStatus($code);
+        $status = $check[0]['status'];
+
+        if($status < 1){
+            return false;
+        }
+
+        $message = $check[0]['errors'];
+        $message = json_decode($message, true);
+
+        if($status > 1){
+            return $message;
+        }
+
+        // Create the archive session variables
+        $_SESSION['zipcompression'] = $message['archive'];
+        unset($message['archive']);
+
+        // Check that the archive has been created
+        $zip = new ZipFolder('', $code);
+        if($zip->checkArchiveExists($code)){
+            // Clean up the download queue and return errors
+            $this->removeItem($code);
+            return $message;
+        }
+        return false;
+    }
+
+    public function isLocked()
+    {
+        return false;
+    }
 }
 ?>
+<?php
+
+chdir(dirname(__FILE__));
+require_once('../../config/dmsDefaults.php');
+require_once('compressionArchiveUtil.inc.php');
+
+/**
+ * The download task provides 2 pieces of functionality. The first is to process the download queue and the second is to "ping" the download queue.
+ * The processing of the download queue will create the archives for the bulk download action and set a flag to indicate their completion.
+ * The "ping" will check if the flag has been set and inform the user that the archive is ready for download.
+ */
+
+$queue = new DownloadQueue();
+
+// Check for a ping then check if the download is finished
+$ping = isset($_POST['ping']) ? $_POST['ping'] : false;
+if($ping == 'ping'){
+
+    $code = $_POST['code'];
+    $status = $queue->isDownloadAvailable($code);
+
+    if($status === false){
+        echo 'wait';
+    }else{
+        $str = '';
+        // display any error messages
+        if(!empty($status)){
+            $str = '<div><b>'._kt('The following errors occurred during the download').': </><br />';
+            $str .= '<table style="padding-top: 5px;" cellspacing="0" cellpadding="5">';
+            foreach ($status as $msg){
+                $str .= '<tr><td style="border: 1px #888 solid;">'.$msg.'</td></tr>';
+            }
+            $str .= '</table></div>';
+        }
+        echo $str;
+    }
+    exit(0);
+}
+
+if($queue->isLocked()){
+    exit(0);
+}
+// Not a ping, process the queue
+$queue->processQueue();
+
+exit(0);
+?>
 \ No newline at end of file
@@ -410,12 +410,12 @@ class KTBulkMoveAction extends KTBulkAction {
         if(is_a($oEntity, 'Folder')) {
             $aDocuments = array();
             $aChildFolders = array();
-            
+
             $oFolder = $oEntity;
             // Get folder id
             $sFolderId = $oFolder->getID();
-            
+
             // Get documents in folder
             $sDocuments = $oFolder->getDocumentIDs($sFolderId);
             $aDocuments = (!empty($sDocuments)) ? explode(',', $sDocuments) : array();
@@ -645,7 +645,7 @@ class KTBulkCopyAction extends KTBulkAction {
                 return PEAR::raiseError(_kt('Document cannot be copied'));
             }
         }
-        
+
         if(is_a($oEntity, 'Folder')) {
             $aDocuments = array();
             $aChildFolders = array();
@@ -1068,18 +1068,42 @@ class KTBrowseBulkExportAction extends KTBulkAction {
                 return PEAR::raiseError(_kt('Document cannot be exported as it is restricted by the workflow.'));
             }
         }
+
         return parent::check_entity($oEntity);
     }
-
     function do_performaction() {
-        $folderName = $this->oFolder->getName();
-        $this->oZip = new ZipFolder($folderName);
-        $res = $this->oZip->checkConvertEncoding();
+        $config = KTConfig::getSingleton();
+        $useQueue = $config->get('export/useDownloadQueue', true);
+
+        // Create the export code
+        $this->sExportCode = KTUtil::randomString();
+        $_SESSION['exportcode'] = $this->sExportCode;
         $folderurl = $this->getReturnUrl();
         $sReturn = sprintf('<p>' . _kt('Return to the original <a href="%s">folder</a>') . "</p>\n", $folderurl);
+        $download_url = KTUtil::addQueryStringSelf("action=downloadZipFile&fFolderId={$this->oFolder->getId()}&exportcode={$this->sExportCode}");
+
+        if($useQueue){
+            $result = parent::do_performaction();
+
+            $url = KTUtil::kt_url() . '/lib/foldermanagement/downloadTask.php';
+
+          	$oTemplating =& KTTemplating::getSingleton();
+          	$oTemplate = $oTemplating->loadTemplate('ktcore/action/bulk_download');
+
+          	$aParams = array(
+                    'return' => $sReturn,
+                    'url' => $url,
+                    'code' => $this->sExportCode,
+                    'download_url' => $download_url
+                );
+            return $oTemplate->render($aParams);
+        }
+
+        $this->oZip = new ZipFolder('', $this->sExportCode);
+        $res = $this->oZip->checkConvertEncoding();
         if(PEAR::isError($res)){
             $this->addErrorMessage($res->getMessage());
@@ -1087,16 +1111,14 @@ class KTBrowseBulkExportAction extends KTBulkAction {
         }
         $this->startTransaction();
-        $oKTConfig =& KTConfig::getSingleton();
-        $this->bNoisy = $oKTConfig->get("tweaks/noisyBulkOperations");
-
-        $this->bNotifications = ($oKTConfig->get('export/enablenotifications', 'on') == 'on') ? true : false;
         $result = parent::do_performaction();
+
         $sExportCode = $this->oZip->createZipFile();
         if(PEAR::isError($sExportCode)){
             $this->addErrorMessage($sExportCode->getMessage());
+            $this->rollbackTransaction();
             return $sReturn;
         }
@@ -1110,23 +1132,28 @@ class KTBrowseBulkExportAction extends KTBulkAction {
         $this->commitTransaction();
-        $url = KTUtil::addQueryStringSelf(sprintf('action=downloadZipFile&fFolderId=%d&exportcode=%s', $this->oFolder->getId(), $sExportCode));
-        $str = sprintf('<p>' . _kt('Your download will begin shortly. If you are not automatically redirected to your download, please click <a href="%s">here</a> ') . "</p>\n", $url);
+        $str = sprintf('<p>' . _kt('Your download will begin shortly. If you are not automatically redirected to your download, please click <a href="%s">here</a> ') . "</p>\n", $download_url);
         $str .= sprintf('<p>' . _kt('Once your download is complete, click <a href="%s">here</a> to return to the original folder') . "</p>\n", $folderurl);
-        //$str .= sprintf("</div></div></body></html>\n");
         $str .= sprintf('<script language="JavaScript">
                 function kt_bulkexport_redirect() {
-                document.location.href = "%s";
+                    document.location.href = "%s";
                 }
-                callLater(2, kt_bulkexport_redirect);
-
-                </script>', $url);
+                callLater(5, kt_bulkexport_redirect);
+                </script>', $download_url);
         return $str;
     }
     function perform_action($oEntity) {
+        $exportCode = $_SESSION['exportcode'];
+        $this->oZip = ZipFolder::get($exportCode);
+
+        $oQueue = new DownloadQueue();
+
+        $config = KTConfig::getSingleton();
+        $useQueue = $config->get('export/useDownloadQueue');
+
         if(is_a($oEntity, 'Document')) {
 			$oDocument = $oEntity;
@@ -1134,19 +1161,12 @@ class KTBrowseBulkExportAction extends KTBulkAction {
 	    		$oDocument->switchToLinkedCore();
 	    	}
-            if ($this->bNoisy) {
-                $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array());
-                $oDocumentTransaction->create();
-            }
-
-            // fire subscription alerts for the downloaded document - if global config is set
-            if($this->bNotifications){
-                $oSubscriptionEvent = new SubscriptionEvent();
-                $oFolder = Folder::get($oDocument->getFolderID());
-                $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder);
-            }
+	    	if($useQueue){
+                DownloadQueue::addItem($this->sExportCode, $this->oFolder->getId(), $oDocument->iId, 'document');
+	    	}else{
+                $oQueue->addDocument($this->oZip, $oDocument->iId);
+	    	}
-            $this->oZip->addDocumentToZip($oDocument);
         }else if(is_a($oEntity, 'Folder')) {
             $aDocuments = array();
@@ -1156,79 +1176,11 @@ class KTBrowseBulkExportAction extends KTBulkAction {
             	$oFolder = $oFolder->getLinkedFolder();
             }
             $sFolderId = $oFolder->getId();
-            $sFolderDocs = $oFolder->getDocumentIDs($sFolderId);
-
-            // Add folder to zip
-            $this->oZip->addFolderToZip($oFolder);
-            if(!empty($sFolderDocs)){
-                $aDocuments = explode(',', $sFolderDocs);
-            }
-
-            // Get all the folders within the current folder
-            $sWhereClause = "parent_folder_ids = '{$sFolderId}' OR
-            parent_folder_ids LIKE '{$sFolderId},%' OR
-            parent_folder_ids LIKE '%,{$sFolderId},%' OR
-            parent_folder_ids LIKE '%,{$sFolderId}'";
-            $aFolderList = $this->oFolder->getList($sWhereClause);
-			$aLinkingFolders = $this->getLinkingEntities($aFolderList);
-            $aFolderList = array_merge($aFolderList,$aLinkingFolders);
-
-            $aFolderObjects = array();
-            $aFolderObjects[$sFolderId] = $oFolder;
-
-            // Export the folder structure to ensure the export of empty directories
-            if(!empty($aFolderList)){
-                foreach($aFolderList as $k => $oFolderItem){
-                    if($oFolderItem->isSymbolicLink()){
-                    	$oFolderItem = $oFolderItem->getLinkedFolder();
-                    }
-                	if(Permission::userHasFolderReadPermission($oFolderItem)){
-	                    // Get documents for each folder
-	                    $sFolderItemId = $oFolderItem->getID();
-	                    $sFolderItemDocs = $oFolderItem->getDocumentIDs($sFolderItemId);
-
-	                    if(!empty($sFolderItemDocs)){
-	                        $aFolderDocs = explode(',', $sFolderItemDocs);
-	                        $aDocuments = array_merge($aDocuments, $aFolderDocs);
-	                    }
-	                    $this->oZip->addFolderToZip($oFolderItem);
-	                    $aFolderObjects[$oFolderItem->getId()] = $oFolderItem;
-                	}
-                }
-            }
-
-            // Add all documents to the export
-            if(!empty($aDocuments)){
-                foreach($aDocuments as $sDocumentId){
-                    $oDocument = Document::get($sDocumentId);
-                 	if($oDocument->isSymbolicLink()){
-	    				$oDocument->switchToLinkedCore();
-	    			}
-	    			if(Permission::userHasDocumentReadPermission($oDocument)){
-
-                        if(!KTWorkflowUtil::actionEnabledForDocument($oDocument, 'ktcore.actions.document.view')){
-                            $this->addErrorMessage($oDocument->getName().': '._kt('Document cannot be exported as it is restricted by the workflow.'));
-                            continue;
-                        }
-
-                        $sDocFolderId = $oDocument->getFolderID();
-                        $oFolder = isset($aFolderObjects[$sDocFolderId]) ? $aFolderObjects[$sDocFolderId] : Folder::get($sDocFolderId);
-
-                        if ($this->bNoisy) {
-                            $oDocumentTransaction = new DocumentTransaction($oDocument, "Document part of bulk export", 'ktstandard.transactions.bulk_export', array());
-                            $oDocumentTransaction->create();
-                        }
-
-                        // fire subscription alerts for the downloaded document
-                        if($this->bNotifications){
-                            $oSubscriptionEvent = new SubscriptionEvent();
-                            $oSubscriptionEvent->DownloadDocument($oDocument, $oFolder);
-                        }
-
-                        $this->oZip->addDocumentToZip($oDocument, $oFolder);
-	    			}
-                }
+            if($useQueue){
+                DownloadQueue::addItem($this->sExportCode, $this->oFolder->getId(), $sFolderId, 'folder');
+            }else{
+                $oQueue->addFolder($this->oZip, $sFolderId);
             }
         }
         return true;
@@ -1237,9 +1189,7 @@ class KTBrowseBulkExportAction extends KTBulkAction {
     function do_downloadZipFile() {
         $sCode = $this->oValidator->validateString($_REQUEST['exportcode']);
-        $folderName = $this->oFolder->getName();
-        $this->oZip = new ZipFolder($folderName, $sCode);
-
+        $this->oZip = new ZipFolder('', $sCode);
         $res = $this->oZip->downloadZipFile($sCode);
         if(PEAR::isError($res)){
@@ -71,20 +71,44 @@ class KTBulkExportAction extends KTFolderAction {
     }
     function do_main() {
-        $folderName = $this->oFolder->getName();
-        $this->oZip = new ZipFolder($folderName);
+        $config = KTConfig::getSingleton();
+        $useQueue = $config->get('export/useDownloadQueue', true);
+
+        // Create the export code
+        $exportCode = KTUtil::randomString();
+        $this->oZip = new ZipFolder('', $exportCode);
         if(!$this->oZip->checkConvertEncoding()) {
             redirect(KTBrowseUtil::getUrlForFolder($this->oFolder));
             exit(0);
         }
-        $oKTConfig =& KTConfig::getSingleton();
-        $bNoisy = $oKTConfig->get("tweaks/noisyBulkOperations");
-        $bNotifications = ($oKTConfig->get('export/enablenotifications', 'on') == 'on') ? true : false;
+        $bNoisy = $config->get("tweaks/noisyBulkOperations");
+        $bNotifications = ($config->get('export/enablenotifications', 'on') == 'on') ? true : false;
-        // Get all folders and sub-folders
         $sCurrentFolderId = $this->oFolder->getId();
+        $url = KTUtil::addQueryStringSelf(sprintf('action=downloadZipFile&fFolderId=%d&exportcode=%s', $sCurrentFolderId, $exportCode));
+        $folderurl = KTBrowseUtil::getUrlForFolder($this->oFolder);
+        $sReturn = '<p>' . _kt('Once your download is complete, click <a href="'.$folderurl.'">here</a> to return to the original folder') . "</p>\n";
+
+        if($useQueue){
+            DownloadQueue::addItem($exportCode, $sCurrentFolderId, $sCurrentFolderId, 'folder');
+
+            $task_url = KTUtil::kt_url() . '/lib/foldermanagement/downloadTask.php';
+
+          	$oTemplating =& KTTemplating::getSingleton();
+          	$oTemplate = $oTemplating->loadTemplate('ktcore/action/bulk_download');
+
+          	$aParams = array(
+                    'return' => $sReturn,
+                    'url' => $task_url,
+                    'code' => $exportCode,
+                    'download_url' => $url
+                );
+            return $oTemplate->render($aParams);
+        }
+
+        // Get all folders and sub-folders
         $sWhereClause = "parent_folder_ids = '{$sCurrentFolderId}' OR
         parent_folder_ids LIKE '{$sCurrentFolderId},%' OR
         parent_folder_ids LIKE '%,{$sCurrentFolderId},%' OR
@@ -173,16 +197,14 @@ class KTBulkExportAction extends KTFolderAction {
             'ip' => Session::getClientIP(),
         ));
-        $url = KTUtil::addQueryStringSelf(sprintf('action=downloadZipFile&fFolderId=%d&exportcode=%s', $this->oFolder->getId(), $sExportCode));
         printf('<p>' . _kt('Your download will begin shortly. If you are not automatically redirected to your download, please click <a href="%s">here</a> ') . "</p>\n", $url);
-        $folderurl = KTBrowseUtil::getUrlForFolder($this->oFolder);
-        printf('<p>' . _kt('Once your download is complete, click <a href="%s">here</a> to return to the original folder') . "</p>\n", $folderurl);
+        print($sReturn);
         printf("</div></div></body></html>\n");
         printf('<script language="JavaScript">
                 function kt_bulkexport_redirect() {
                     document.location.href = "%s";
                 }
-                callLater(1, kt_bulkexport_redirect);
+                callLater(2, kt_bulkexport_redirect);
                 </script>', $url);
@@ -230,8 +252,7 @@ class KTBulkExportAction extends KTFolderAction {
     function do_downloadZipFile() {
         $sCode = $this->oValidator->validateString($_REQUEST['exportcode']);
-        $folderName = $this->oFolder->getName();
-        $this->oZip = new ZipFolder($folderName);
+        $this->oZip = new ZipFolder('', $sCode);
         $res = $this->oZip->downloadZipFile($sCode);
@@ -287,7 +287,9 @@ INSERT INTO `config_settings` VALUES
 (112, 'urls', 'Var Directory', 'The path to the var directory.', 'varDirectory', 'default', '${fileSystemRoot}/var', 'string', NULL, 1),
 (113, 'tweaks','Increment version on rename','Defines whether to update the version number if a document filename is changed/renamed.','incrementVersionOnRename','default','true','boolean',NULL,1),
 (114, 'ui', 'System URL', 'The system url, used in the main logo.', 'systemUrl', 'default', 'http://www.knowledgetree.com', 'string', '', 1),
-(115, 'ldapAuthentication', 'Allow Moving Users in LDAP/AD', 'Moving users around within the LDAP or Active Directory structure will cause failed logins for these users. When this setting is enabled, a failed login will trigger a search for the user using their sAMAccountName setting and update their authentication details.', 'enableLdapUpdate', 'default', 'false', 'boolean', NULL, 1);
+(115, 'ldapAuthentication', 'Allow Moving Users in LDAP/AD', 'Moving users around within the LDAP or Active Directory structure will cause failed logins for these users. When this setting is enabled, a failed login will trigger a search for the user using their sAMAccountName setting and update their authentication details.', 'enableLdapUpdate', 'default', 'false', 'boolean', NULL, 1),
+(116, 'export', 'Use External Zip Binary', 'Utilises the external zip binary for compressing archives. The default is to use the PEAR archive class.', 'useBinary', 'default', 'false', 'boolean', NULL, 1),
+(117, 'export', 'Use Bulk Download Queue', 'The bulk download can be large and can prevent normal browsing. The download queue performs the bulk downloads in the background.', 'useDownloadQueue', 'default', 'true', 'boolean', NULL, 1);
 /*!40000 ALTER TABLE `config_settings` ENABLE KEYS */;
 UNLOCK TABLES;
@@ -1370,7 +1372,8 @@ INSERT INTO `scheduler_tasks` VALUES
 (7,'Cleanup Temporary Directory','search2/bin/cronCleanup.php','',0,'1min','2007-10-01 00:00:00',NULL,0,'enabled'),
 (8,'Disk Usage and Folder Utilisation Statistics','plugins/housekeeper/bin/UpdateStats.php','',0,'5mins','2007-10-01 00:00:00',NULL,0,'enabled'),
 (9,'Refresh Index Statistics','search2/bin/cronIndexStats.php','',0,'1min','2007-10-01',NULL,0,'enabled'),
-(10,'Refresh Resource Dependancies','search2/bin/cronResources.php','',0,'1min','2007-10-01',NULL,0,'enabled');
+(10,'Refresh Resource Dependancies','search2/bin/cronResources.php','',0,'1min','2007-10-01',NULL,0,'enabled'),
+(11,'Bulk Download Queue','lib/foldermanagement/downloadTask.php','',0,'1min','2007-10-01',NULL,0,'system');
 /*!40000 ALTER TABLE `scheduler_tasks` ENABLE KEYS */;
 UNLOCK TABLES;
@@ -639,6 +639,22 @@ CREATE TABLE `download_files` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 --
+-- Table structure for table `download_queue`
+--
+
+CREATE TABLE `download_queue` (
+	`code` char(16) NOT NULL,
+	`folder_id` int(11) NOT NULL,
+	`object_id` int(11) NOT NULL,
+	`object_type` enum('document', 'folder') NOT NULL default 'folder',
+	`user_id` int(11) NOT NULL,
+	`date_added` timestamp NOT NULL default CURRENT_TIMESTAMP,
+	`status` tinyint(4) NOT NULL default 0,
+	`errors` mediumtext,
+	INDEX (`code`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+--
 -- Table structure for table `field_behaviour_options`
 --
+--
+-- Table structure for table `download_queue`
+--
+
+CREATE TABLE `download_queue` (
+	`code` char(16) NOT NULL,
+	`folder_id` int(11) NOT NULL,
+	`object_id` int(11) NOT NULL,
+	`object_type` enum('document', 'folder') NOT NULL default 'folder',
+	`user_id` int(11) NOT NULL,
+	`date_added` timestamp NOT NULL default CURRENT_TIMESTAMP,
+	`status` tinyint(4) NOT NULL default 0,
+	`errors` mediumtext,
+	INDEX (`code`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+INSERT INTO `config_settings` (group_name, display_name, description, item, value, default_value, type, options, can_edit)
+VALUES ('export', 'Use External Zip Binary', 'Utilises the external zip binary for compressing archives. The default is to use the PEAR archive class.', 'useBinary', 'default', 'false', 'boolean', NULL, 1),
+('export', 'Use Bulk Download Queue', 'The bulk download can be large and can prevent normal browsing. The download queue performs the bulk downloads in
+the background.', 'useDownloadQueue', 'default', 'true', 'boolean', NULL, 1);
+
+INSERT INTO `scheduler_tasks` (task, script_url, frequency, run_time, status)
+VALUES ('Bulk Download Queue','lib/foldermanagement/downloadTask.php','1min','2007-10-01','system');
 \ No newline at end of file
+<p style='margin-bottom: 10px;'>
+The documents you have selected are being compressed and archived, once the archive has been created a download link will be displayed.
+<br /><b>Please do not close the page.</b>
+</p>
+
+
+<div id='download_link' style='visibility: hidden'>
+<p style='margin-bottom: 5px;'>
+Click <a href="{$download_url}">here</a> to download the archive. {$str}
+</p>
+<p style='margin-bottom: 10px;'>
+{$return}
+<br />
+</p>
+</div>
+
+
+<script type='text/javascript'>
+{literal}
+function ping()
+{
+    Ext.Ajax.request({
+        url: {/literal}'{$url}'{literal},
+        success: function(response) {
+            if(response.responseText == 'wait'){
+                setTimeout("ping()", 2000);
+                return;
+            }
+
+            var div = document.getElementById('download_link');
+            div.style.visibility = 'visible';
+            div.innerHTML = div.innerHTML + response.responseText;
+        },
+        failure: function(response) {
+            alert({/literal}{i18n}'There was an error connecting to the server. Please refresh the page.'{/i18n}{literal});
+        },
+        params: {
+            ping: 'ping',
+            code: {/literal}'{$code}'{literal}
+        }
+    });
+
+}
+{/literal}
+
+ping();
+</script>
 \ No newline at end of file