/*
 * RHQ Management Platform
 * Copyright (C) 2005-2010 Red Hat, Inc.
 * All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation, and/or the GNU Lesser
 * General Public License, version 2.1, also as published by the Free
 * Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License and the GNU Lesser General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License
 * and the GNU Lesser General Public License along with this program;
 * if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

package org.rhq.core.util.updater;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.rhq.core.util.MessageDigestGenerator;
import org.rhq.core.util.ZipUtil;
import org.rhq.core.util.ZipUtil.ZipEntryVisitor;
import org.rhq.core.util.file.FileUtil;
import org.rhq.core.util.stream.StreamCopyDigest;
import org.rhq.core.util.stream.StreamUtil;

/**
 * This deploys a bundle of files within a zip archive to a managed directory.
 * 
 * This follows rpm-like rules when updating a deployment and files already exist
 * in previous deployments that need to be re-installed in an updated deployment.
 * The rules are as follows, where the first three columns represent a file's
 * hashcode:
 * <table>
 * <tr><th>ORIGINAL</th><th>CURRENT</th><th>NEW</th><th>What To Do...</th></tr>
 * <tr><td>X</td><td>X</td><td>X</td><td>New file is installed over current*</td></tr>
 * <tr><td>X</td><td>X</td><td>Y</td><td>New file is installed over current</td></tr>
 * <tr><td>X</td><td>Y</td><td>X</td><td>Current file is left as-is</td></tr>
 * <tr><td>X</td><td>Y</td><td>Y</td><td>New file is installed over current*</td></tr>
 * <tr><td>X</td><td>Y</td><td>Z</td><td>New file is installed over current, current is backed up</td></tr>
 * <tr><td>none</td><td>?</td><td>?</td><td>New file is installed over current, current is backed up</td></tr>
 * <tr><td>X</td><td>none</td><td>?</td><td>New file is installed</td></tr>
 * <tr><td>?</td><td>?</td><td>none</td><td>Current file is backed up and deleted</td></tr>
 * </table>
 * (*) denotes that the current file could have been left as-is. If, in the future, we can
 * provide Java with a way to change file permissions or ownership, we would want to
 * copy the new file over the current file and perform the chown/chmod.
 * 
 * Here you can see that there is only one non-trivial case where the new file is <b>not</b> installed, and that
 * is when the original file and the new file have the same hashcode (i.e. was not changed) but
 * the current version of that file was changed (in the trivial case, where there is no new file, nothing is
 * installed and the current file is uninstalled). In this case, since the original file and the new
 * file are identical, the file with the modifications made to the original version (called the "current" version)
 * is presumed to still be valid for the new version, thus we leave the current, modified file in place.
 * 
 * In the case where there is ambiguity as to what the right thing to do is, the current file is
 * backed up prior to being overwritten with the new file. This can occur in two cases: the first
 * is when the current file is a modified version of the original and the new file is different
 * from the original; the second is when there was no original file, but there is now a current
 * file on disk and a new file to be installed. In either case, the current file is backed up
 * before the new file is copied over top the current file. Files that need to be
 * backed up will have its backup copied to new deployment's metadata directory, under
 * the backup subdirectory called {@link DeploymentsMetadata#BACKUP_DIR}. If a file
 * that needs to be backed up is referred to via an absolute path (that is, outside the destination
 * directory), it will be copied to the {@link DeploymentsMetadata#EXT_BACKUP_DIR} directory found
 * in the metadata directory. 
 * 
 * @author John Mazzitelli
 */
public class Deployer {
    private final Log log = LogFactory.getLog(Deployer.class);

    private final DeploymentData deploymentData;
    private final DeploymentsMetadata deploymentsMetadata;

    /**
     * Constructors that prepares this object to deploy content to a destination on the local file system.
     *  
     * @param deploymentData the data needed to know what to do for this deployment
     */
    public Deployer(DeploymentData deploymentData) {
        if (deploymentData == null) {
            throw new IllegalArgumentException("deploymentData == null");
        }

        this.deploymentData = deploymentData;
        this.deploymentsMetadata = new DeploymentsMetadata(deploymentData.getDestinationDir());
        return;
    }

    /**
     * @return information about the particular deployment that this deployer will install.
     */
    public DeploymentData getDeploymentData() {
        return deploymentData;
    }

    /** 
     * @return <code>true</code> if the deployer is to install the deployment data in a directory
     *         that already has a managed deployment in it. <code>false</code> if there is currently
     *         no managed deployments in the destination directory.
     */
    public boolean isDestinationDirectoryManaged() {
        return deploymentsMetadata.isManaged();
    }

    /**
     * Convienence method that is equivalent to {@link #deploy(DeployDifferences, boolean) deploy(diff, false)}.
     * @see #deploy(DeployDifferences, boolean) 
     */
    public FileHashcodeMap deploy(DeployDifferences diff) throws Exception {
        return deploy(diff, false, false);
    }

    /**
     * Convienence method that is equivalent to {@link #deploy(DeployDifferences, boolean) deploy(diff, true)}.
     * @see #deploy(DeployDifferences, boolean) 
     */
    public FileHashcodeMap dryRun(DeployDifferences diff) throws Exception {
        return deploy(diff, false, true);
    }

    /**
     * Deploys all files to their destinations. Everything this method has to do is determined
     * by the data passed into this object's constructor.
     * 
     * This method allows one to ask for a dry run to be performed; meaning don't actually change
     * the file system, just populate the <code>diff</code> object with what would have been done.
     * 
     * The caller can ask that an existing deployment directory be cleaned (i.e. wiped from the file system)
     * before the new deployment is laid down. This is useful if there are ignored files/directories
     * that would be left alone as-is, had a clean not been requested. Note that the <code>clean</code>
     * parameter is ignored if a dry run is being requested.
     * 
     * @param diff this method will populate this object with information about what
     *             changed on the file system due to this deployment
     * @param clean if <code>true</code>, the caller is telling this method to first wipe clean
     *              any existing deployment files found in the destination directory before installing the
     *              new deployment. Note that this will have no effect if <code>dryRun</code>
     *              is <code>true</code> since a dry run by definition never affects the file system.
     * @param dryRun if <code>true</code>, the file system won't actually be changed; however,
     *               the <code>diff</code> object will still be populated with information about
     *               what would have occurred on the file system had it not been a dry run
     * @return file/hashcode information for all files that were deployed
     * @throws Exception if the deployment failed for some reason
     */
    public FileHashcodeMap deploy(DeployDifferences diff, boolean clean, boolean dryRun) throws Exception {
        FileHashcodeMap map = null;

        // fail-fast if we don't have enough disk space
        checkDiskUsage();

        if (!this.deploymentsMetadata.isManaged()) {
            // the destination dir has not been used to deploy a bundle yet, therefore, this is the first deployment
            map = performInitialDeployment(diff, dryRun);
        } else {
            // we have metadata in the destination directory, therefore, this deployment is updating a current one
            map = performUpdateDeployment(diff, clean, dryRun);
        }

        return map;
    }

    /**
     * This will first perform a deploy (e.g. {@link #deploy(DeployDifferences, boolean, boolean) deploy(diff, clean, dryRun)})
     * and then, if there are backup files from the previous deployment, those backup files will be restored to their
     * original locations.
     * 
     * This is useful when you want to "undeploy" something where "undeploy" infers you want to go back to
     * how the file system looked previously to a subsequent deployment (the one this method is being told
     * to "redeploy"), including manual changes that were made over top the previous deployment.
     * 
     * For example, suppose you deployed deployment ID #1 and then the user manually changed some files
     * within that #1 deployment. Later on, you deploy deployment ID #2 (which will not only deploy
     * #2's files but will also backup the files that were manually changed within deployment #1).
     * You find that deployment #2 is bad and you want to revert back to how the file system looked
     * previously. You could opt to rollback to deployment ID #1 minus those manual changes - to do
     * this you simply {@link #deploy(DeployDifferences) deploy} #1 again. However, if you want to
     * go back to the previous content including those manual changes, you first deploy #1 and
     * then restore the backup files - essentially overlaying the manual changes over top #1 files. This
     * method accomplishes that latter task.
     *
     * @param diff see {@link #deploy(DeployDifferences, boolean, boolean)}
     * @param clean see {@link #deploy(DeployDifferences, boolean, boolean)}
     * @param dryRun see {@link #deploy(DeployDifferences, boolean, boolean)}
     * @return see {@link #deploy(DeployDifferences, boolean, boolean)}
     * @throws Exception if either the deployment or backup file restoration failed
     */
    public FileHashcodeMap redeployAndRestoreBackupFiles(DeployDifferences diff, boolean clean, boolean dryRun)
        throws Exception {
        FileHashcodeMap map = null;

        // fail-fast if we don't have enough disk space
        checkDiskUsage();

        if (!this.deploymentsMetadata.isManaged()) {
            // the destination dir has not been used to deploy a bundle yet, therefore, this is the first deployment
            map = performInitialDeployment(diff, dryRun);
        } else {
            // we have metadata in the destination directory, therefore, this deployment is updating a current one.
            // First get the ID of the currently existing deployment - this is where our backup files exist.
            // Then we update the current deployment with the new deployment (which actually should be restoring to the previous one).
            // Finally, we restore the backup files into the new deployment, overlaying them over top the new deployment.
            int id = this.deploymentsMetadata.getCurrentDeploymentProperties().getDeploymentId();
            map = performUpdateDeployment(diff, clean, dryRun);
            restoreBackupFiles(id, map, diff, dryRun);
            if (!dryRun) {
                // if we restored one or more files, we need to persist the new deployment hashcode data with the restored hashcodes
                if (!diff.getRestoredFiles().isEmpty()) {
                    this.deploymentsMetadata.setCurrentDeployment(this.deploymentData.getDeploymentProps(), map, false);
                }
            }
        }

        return map;
    }

    /**
     * Returns an estimated amount of disk space the deployment will need if it gets installed.
     * @return information on the estimated disk usage
     * @throws Exception if cannot determine the estimated disk usage
     */
    public DeploymentDiskUsage estimateDiskUsage() throws Exception {
        final DeploymentDiskUsage diskUsage = new DeploymentDiskUsage();

        File partition = this.deploymentData.getDestinationDir();
        long usableSpace = partition.getUsableSpace();
        while (usableSpace == 0L && partition != null) {
            partition = partition.getParentFile();
            if (partition != null) {
                usableSpace = partition.getUsableSpace();
            }
        }

        diskUsage.setMaxDiskUsable(usableSpace);

        Set<File> zipFiles = this.deploymentData.getZipFiles();
        for (File zipFile : zipFiles) {
            ZipUtil.walkZipFile(zipFile, new ZipEntryVisitor() {
                public boolean visit(ZipEntry entry, ZipInputStream stream) throws Exception {
                    if (!entry.isDirectory()) {
                        final long size = entry.getSize();
                        diskUsage.increaseDiskUsage(size > 0 ? size : 0);
                        diskUsage.incrementFileCount();
                    }
                    return true;
                }
            });
        }

        Map<File, File> rawFiles = this.deploymentData.getRawFiles();
        for (File rawFile : rawFiles.keySet()) {
            diskUsage.increaseDiskUsage(rawFile.length());
            diskUsage.incrementFileCount();
        }

        return diskUsage;
    }

    /**
     * This will get an {@link #estimateDiskUsage() estimate} of how much disk space the deployment will need
     * and compare it to the amount of estimated disk space is currently usable. If there does not appear to be
     * enough usable disk space to fit the deployment, this method will thrown an exception. Otherwise, this
     * method will simply return normally.
     * 
     * This can be used to fail-fast a deployment - there is no need to process the deployment if there is not
     * enough disk space to start with.
     * 
     * @throws Exception if there does not appear to be enough disk space to store the deployment content
     */
    public void checkDiskUsage() throws Exception {
        DeploymentDiskUsage usage;

        try {
            usage = estimateDiskUsage();
        } catch (Exception e) {
            debug("Cannot estimate disk usage - will assume there is enough space and will continue. Cause: ", e);
            return;
        }

        debug("Estimated disk usage for this deployment is [", usage.getDiskUsage(), "] bytes (file count=[", usage
            .getFileCount(), "]). The maximum disk space currently usable is estimated to be [", usage
            .getMaxDiskUsable(), "] bytes.");

        if (usage.getDiskUsage() > usage.getMaxDiskUsable()) {
            throw new Exception(
                "There does not appear to be enough disk space for this deployment. Its estimated disk usage ["
                    + usage.getDiskUsage() + "] is larger than the estimated amount of usable disk space ["
                    + usage.getMaxDiskUsable() + "].");
        }

        return;
    }

    private FileHashcodeMap performInitialDeployment(DeployDifferences diff, boolean dryRun) throws Exception {
        FileHashcodeMap newFileHashcodeMap = extractZipAndRawFiles(new HashMap<String, String>(0), diff, dryRun);

        // this is an initial deployment, so we know every file is new - tell our diff about them all
        if (diff != null) {
            diff.addAddedFiles(newFileHashcodeMap.keySet());
        }

        debug("Initial deployment finished. dryRun=", dryRun);
        return newFileHashcodeMap;
    }

    private FileHashcodeMap performUpdateDeployment(DeployDifferences diff, boolean clean, boolean dryRun)
        throws Exception {
        debug("Analyzing original, current and new files as part of update deployment. dryRun=", dryRun);

        // NOTE: remember that data that goes in diff are only things affecting the current file set such as:
        //       * if a current file will change
        //       * if a current file will be deleted
        //       * if a new file is added to the filesystem
        //       * if a current file is ignored in the latest rescan
        //       * if a file is realized on the filesystem before its stored on the file system
        //       * if a current file is backed up

        FileHashcodeMap original = this.deploymentsMetadata.getCurrentDeploymentFileHashcodes();
        ChangesFileHashcodeMap current = original.rescan(this.deploymentData.getDestinationDir(), this.deploymentData
            .getIgnoreRegex(), this.deploymentData.isManageRootDir());
        FileHashcodeMap newFiles = getNewDeploymentFileHashcodeMap();

        if (current.getUnknownContent() != null) {
            throw new Exception("Failed to properly rescan the current deployment: " + current.getUnknownContent());
        }

        if (diff != null) {
            diff.addIgnoredFiles(current.getIgnored());
            for (Map.Entry<String, String> entry : current.entrySet()) {
                String currentPath = entry.getKey();
                String currentHashcode = entry.getValue();
                if (currentHashcode.equals(FileHashcodeMap.DELETED_FILE_HASHCODE)) {
                    if (newFiles.containsKey(currentPath)) {
                        diff.addAddedFile(currentPath);
                    }
                } else if (!newFiles.containsKey(currentPath)) {
                    diff.addDeletedFile(currentPath);
                } else if (!newFiles.get(currentPath).equals(currentHashcode)) {
                    if (!newFiles.get(currentPath).equals(original.get(currentPath))) {
                        diff.addChangedFile(currentPath);
                    }
                }
            }
            for (Map.Entry<String, String> entry : newFiles.entrySet()) {
                if (!current.containsKey(entry.getKey())) {
                    diff.addAddedFile(entry.getKey());
                }
            }
        }

        // backup the current file if either of these are true:
        // * if there is a current file but there was no original file (current.getAdditions())
        // * if the current, original and new files are all different from one another (current.getChanges() compared to new)
        //
        // do nothing (do not backup the current file and do not copy the new file to disk) if:
        // * the original file is not the same as the current file (current.getChanges()) but is the same as the new file
        Set<String> currentFilesToBackup = new HashSet<String>(current.getAdditions().keySet());
        Map<String, String> currentFilesToLeaveAlone = new HashMap<String, String>();
        Set<String> currentFilesToDelete;

        for (Map.Entry<String, String> changed : current.getChanges().entrySet()) {
            String changedFilePath = changed.getKey();
            String newHashcode = newFiles.get(changedFilePath);
            if (newHashcode != null) {
                // Remember, original hash will never be the same as the changed hash (that's why it is considered changed!).
                // If the new file hash is the same as the original, it means the user changed the file but that change
                // should be compatible with the new deployment (this is assumed, but if the new file didn't alter the original file,
                // presumably the new software is backward compatible and should accept the changed original as a valid file).
                // Now however in this case we mark the changed file path with the *original* hashcode - in effect, we continue
                // along the knowlege of the original hashcode (which is the same as the new) thus continuing the knowledge
                // that a file is different from the deployment distribution.
                // If the new is different from the original and if the new is different than the current, we need to backup the
                // current because we will be overwriting the current file with the new.
                String changedFileHashcode = changed.getValue();
                String originalFileHashcode = original.get(changedFilePath);
                if (newHashcode.equals(originalFileHashcode)) {
                    currentFilesToLeaveAlone.put(changedFilePath, originalFileHashcode);
                } else if (!newHashcode.equals(changedFileHashcode)) {
                    currentFilesToBackup.add(changedFilePath);
                }
            } else {
                // the current file is a changed version of original, but there is no new file
                // we should back this up since it will be deleted but was changed from the original for some reason
                currentFilesToBackup.add(changedFilePath);
            }
        }

        // done with this, allow GC to reclaim memory
        original = null;

        // the rescan can only find things in the dest dir.
        // if the new deployment introduced new files with absolute paths ('new' meaning they
        // were not in the original deployment), we need to see if that new absolute path
        // already exists; if it does, we need to back that file up.
        Set<String> newNotFoundByRescan = new HashSet<String>(newFiles.keySet());
        newNotFoundByRescan.removeAll(current.keySet());
        for (String newFileNotScanned : newNotFoundByRescan) {
            File newFileNotScannedFile = new File(newFileNotScanned);
            if (newFileNotScannedFile.isAbsolute()) {
                if (newFileNotScannedFile.exists()) {
                    currentFilesToBackup.add(newFileNotScanned);
                    if (diff != null) {
                        diff.removeAddedFile(newFileNotScanned);
                        diff.addChangedFile(newFileNotScanned);
                    }
                }
            }
        }

        // done with this, allow GC to reclaim memory
        newNotFoundByRescan = null;

        // now remove all the new files from the current map - what's left are files that exist that are not in the new deployment
        // thus, those files left in current are those that need to be deleted from disk
        currentFilesToDelete = current.keySet(); // the set is backed by the map, changes to it affect the map
        currentFilesToDelete.removeAll(newFiles.keySet());
        currentFilesToDelete.removeAll(current.getDeletions().keySet()); // these are already deleted, no sense trying to delete them again

        // remember what files were skipped so we don't delete them during our purge below (only care about this if we are going to 'clean')
        Set<String> skippedFiles = (clean) ? current.getSkipped() : null;

        // don't use this anymore - its underlying key set has been altered and this no longer is the full current files
        current = null;

        // we now know what to do:
        // 1. backup the files in currentFilesToBackup
        // 2. delete the files in currentFilesToDelete
        // 3. copy all new files in newFiles except for those files that are also in currentFilesToLeaveAlone

        // 1. backup the files we want to retain for the admin to review
        if (!currentFilesToBackup.isEmpty()) {
            DeploymentProperties props = this.deploymentData.getDeploymentProps(); // changed from: deploymentsMetadata.getCurrentDeploymentProperties
            int backupDeploymentId = props.getDeploymentId();
            debug("Backing up files as part of update deployment. dryRun=", dryRun);
            for (String fileToBackupPath : currentFilesToBackup) {
                backupFile(diff, backupDeploymentId, fileToBackupPath, dryRun);
            }
        }

        // 2. delete the obsolete files
        if (!currentFilesToDelete.isEmpty()) {
            debug("Deleting obsolete files as part of update deployment. dryRun=", dryRun);
            for (String fileToDeletePath : currentFilesToDelete) {
                File doomedFile = new File(fileToDeletePath);
                if (!doomedFile.isAbsolute()) {
                    doomedFile = new File(this.deploymentData.getDestinationDir(), fileToDeletePath);
                }
                boolean deleted;
                if (!dryRun) {
                    deleted = doomedFile.delete();
                } else {
                    deleted = true;
                }
                if (deleted) {
                    debug("Deleted obsolete file [", doomedFile, "]. dryRun=", dryRun);
                } else {
                    // TODO: what should we do? is it a major failure if we can't remove obsolete files?                
                    debug("Failed to delete obsolete file [", doomedFile, "]");
                    if (diff != null) {
                        diff.addError(fileToDeletePath, "File [" + doomedFile.getAbsolutePath() + "] did not delete");
                    }
                }
            }
        }

        // 3. copy all new files except for those to be kept as-is (perform a clean first, if requested)
        if (clean) {
            debug("Cleaning the existing deployment's files found in the destination directory. dryRun=", dryRun);
            if (!dryRun) {
                purgeFileOrDirectory(this.deploymentData.getDestinationDir(), skippedFiles, 0, false);
            }
        }
        diff.setCleaned(clean);

        debug("Copying new files as part of update deployment. dryRun=", dryRun);
        FileHashcodeMap newFileHashCodeMap = extractZipAndRawFiles(currentFilesToLeaveAlone, diff, dryRun);

        debug("Update deployment finished. dryRun=", dryRun);
        return newFileHashCodeMap;
    }

    private void backupFile(DeployDifferences diff, int deploymentId, final String fileToBackupPath, boolean dryRun)
        throws Exception {

        File bakFile;

        // we need to play some games if we are on windows and a drive letter is specified
        boolean isWindows = (File.separatorChar == '\\');
        StringBuilder fileToBackupPathNoDriveLetter = null;
        String driveLetter = null;

        if (isWindows) {
            fileToBackupPathNoDriveLetter = new StringBuilder(fileToBackupPath);
            driveLetter = FileUtil.stripDriveLetter(fileToBackupPathNoDriveLetter);
        }

        File fileToBackup = new File(fileToBackupPath);
        if (fileToBackup.isAbsolute()) {
            File backupDir = this.deploymentsMetadata.getDeploymentExternalBackupDirectory(deploymentId);
            if (isWindows && driveLetter != null) {
                String dirName = this.deploymentsMetadata.getExternalBackupDirectoryNameForWindows(driveLetter);
                backupDir = new File(backupDir, dirName);
                bakFile = new File(backupDir, fileToBackupPathNoDriveLetter.toString());
            } else {
                bakFile = new File(backupDir, fileToBackupPath);
            }
        } else {
            File backupDir = this.deploymentsMetadata.getDeploymentBackupDirectory(deploymentId);
            if (isWindows && driveLetter != null) {
                StringBuilder destDirAbsPathBuilder = new StringBuilder(this.deploymentData.getDestinationDir()
                    .getAbsolutePath());
                String destDirDriveLetter = FileUtil.stripDriveLetter(destDirAbsPathBuilder);
                if (destDirDriveLetter == null || driveLetter.equals(destDirDriveLetter)) {
                    bakFile = new File(backupDir, fileToBackupPath);
                    fileToBackup = new File(this.deploymentData.getDestinationDir(), fileToBackupPathNoDriveLetter
                        .toString());
                } else {
                    throw new Exception("Cannot backup relative path [" + fileToBackupPath
                        + "] whose drive letter is different than the destination directory ["
                        + this.deploymentData.getDestinationDir().getAbsolutePath() + "]");
                }
            } else {
                bakFile = new File(backupDir, fileToBackupPath);
                fileToBackup = new File(this.deploymentData.getDestinationDir(), fileToBackupPath);
            }
        }

        if (!dryRun) {
            bakFile.getParentFile().mkdirs();
            FileUtil.copyFile(fileToBackup, bakFile);
        }

        debug("Backed up file [", fileToBackup, "] to [", bakFile, "]. dryRun=", dryRun);

        if (diff != null) {
            diff.addBackedUpFile(fileToBackupPath, bakFile.getAbsolutePath());
        }

        return;
    }

    private FileHashcodeMap extractZipAndRawFiles(Map<String, String> currentFilesToLeaveAlone, DeployDifferences diff,
        boolean dryRun) throws Exception {

        // NOTE: right now, this only adds to the "realized" set of files is diff, no need to track "added" or "changed" here
        FileHashcodeMap newFileHashCodeMap = new FileHashcodeMap();

        // get information about the source dir - will be needed if we were told to not explode a zip
        String sourceDirAbsPath = this.deploymentData.getSourceDir().getAbsolutePath();
        int sourceDirLength = sourceDirAbsPath.length();

        // extract all zip files
        ExtractorZipFileVisitor visitor;
        for (File zipFile : this.deploymentData.getZipFiles()) {
            Boolean exploded = this.deploymentData.getZipsExploded().get(zipFile);
            if (exploded == null) {
                exploded = Boolean.TRUE; // the default is to explode the archive
            }

            debug("Extracting zip [", zipFile, "] entries. exploded=", exploded, ", dryRun=", dryRun);

            Pattern realizeRegex = null;
            if (this.deploymentData.getZipEntriesToRealizeRegex() != null) {
                realizeRegex = this.deploymentData.getZipEntriesToRealizeRegex().get(zipFile);
            }

            if (exploded.booleanValue()) {
                // EXPLODED
                visitor = new ExtractorZipFileVisitor(this.deploymentData.getDestinationDir(), realizeRegex,
                    this.deploymentData.getTemplateEngine(), currentFilesToLeaveAlone.keySet(), diff, dryRun);
                ZipUtil.walkZipFile(zipFile, visitor);
                newFileHashCodeMap.putAll(visitor.getFileHashcodeMap()); // exploded into individual files
            } else {
                // COMPRESSED

                // Note: there is a requirement that all zip files must be located in the sourceDir - this is why. We need
                // the path of the zip relative to the source dir so we can copy it to the same relative location
                // under the destination dir. Without doing this, if the zip is in a subdirectory, we won't know where to
                // put it under the destination dir. 
                String zipRelativePath = zipFile.getAbsolutePath().substring(sourceDirLength);
                if (zipRelativePath.startsWith("/") || zipRelativePath.startsWith("\\")) {
                    zipRelativePath = zipRelativePath.substring(1);
                }
                File compressedFile = new File(this.deploymentData.getDestinationDir(), zipRelativePath);

                if (this.deploymentData.getTemplateEngine() != null && realizeRegex != null) {
                    // we need to explode it to perform the realization of templatized variables
                    // TODO: can we do this in another tmp location and build the zip in the dest dir?
                    visitor = new ExtractorZipFileVisitor(this.deploymentData.getDestinationDir(), realizeRegex,
                        this.deploymentData.getTemplateEngine(), currentFilesToLeaveAlone.keySet(), diff, dryRun);
                    ZipUtil.walkZipFile(zipFile, visitor);
                    // we have to compress the file again - our new compressed file will have the new realized files in them
                    if (!dryRun) {
                        createZipFile(compressedFile, this.deploymentData.getDestinationDir(), visitor
                            .getFileHashcodeMap());
                    }
                }

                // Copy the archive to the destination dir if we need to. Generate its hashcode and add it to the new file hashcode map
                MessageDigestGenerator hashcodeGenerator = new MessageDigestGenerator();
                String compressedFileHashcode;
                if (!dryRun) {
                    if (!compressedFile.exists()) {
                        if (compressedFile.getParentFile() != null) {
                            compressedFile.getParentFile().mkdirs();
                        }
                        FileUtil.copyFile(zipFile, compressedFile);
                    }
                    compressedFileHashcode = hashcodeGenerator.calcDigestString(compressedFile);
                } else {
                    // use source zip for hash - should be the same as the would-be compressed file since we aren't realizing files in it
                    compressedFileHashcode = hashcodeGenerator.calcDigestString(zipFile);
                }
                newFileHashCodeMap.put(zipRelativePath, compressedFileHashcode);
            }
        }

        // copy all raw files
        StreamCopyDigest copyDigester = new StreamCopyDigest();
        for (Map.Entry<File, File> rawFile : this.deploymentData.getRawFiles().entrySet()) {
            // determine where the original file is and where it needs to go
            File currentLocationFile = rawFile.getKey();
            File newLocationFile = rawFile.getValue();
            String newLocationPath = rawFile.getValue().getPath();
            newLocationPath = newFileHashCodeMap.convertPath(newLocationPath);
            if (currentFilesToLeaveAlone != null && currentFilesToLeaveAlone.containsKey(newLocationPath)) {
                continue;
            }
            if (!newLocationFile.isAbsolute()) {
                newLocationFile = new File(this.deploymentData.getDestinationDir(), newLocationFile.getPath());
            }

            if (!dryRun) {
                File newLocationParentDir = newLocationFile.getParentFile();
                newLocationParentDir.mkdirs();
                if (!newLocationParentDir.isDirectory()) {
                    throw new Exception("Failed to create new parent directory for raw file [" + newLocationFile + "]");
                }
            }

            String hashcode;

            boolean realize = false;
            if (this.deploymentData.getRawFilesToRealize() != null) {
                realize = this.deploymentData.getRawFilesToRealize().contains(currentLocationFile);
            }

            if (realize) {
                debug("Realizing file [", currentLocationFile, "] to [", newLocationFile, "]. dryRun=", dryRun);

                // this entry needs to be realized, do it now in-memory (we assume realizable files will not be large)
                // note: tempateEngine will never be null if we got here
                FileInputStream in = new FileInputStream(currentLocationFile);
                byte[] rawFileContent = StreamUtil.slurp(in);
                String content = this.deploymentData.getTemplateEngine().replaceTokens(new String(rawFileContent));

                if (diff != null) {
                    diff.addRealizedFile(newLocationPath, content);
                }

                // now write the realized content to the filesystem
                byte[] bytes = content.getBytes();

                if (!dryRun) {
                    BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(newLocationFile));
                    try {
                        out.write(bytes);
                    } finally {
                        out.close();
                    }
                }

                MessageDigestGenerator hashcodeGenerator = copyDigester.getMessageDigestGenerator();
                hashcodeGenerator.add(bytes);
                hashcode = hashcodeGenerator.getDigestString();
            } else {
                debug("Copying raw file [", currentLocationFile, "] to [", newLocationFile, "]. dryRun=", dryRun);

                FileInputStream in = new FileInputStream(currentLocationFile);
                try {
                    if (!dryRun) {
                        BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(newLocationFile));
                        try {
                            hashcode = copyDigester.copyAndCalculateHashcode(in, out);
                        } finally {
                            out.close();
                        }
                    } else {
                        hashcode = MessageDigestGenerator.getDigestString(in);
                    }
                } finally {
                    in.close();
                }
            }

            // remember where the file is now and what its hashcode is
            if (rawFile.getValue().isAbsolute()) {
                newFileHashCodeMap.put(newLocationFile.getAbsolutePath(), hashcode);
            } else {
                newFileHashCodeMap.put(newLocationPath, hashcode);
            }
        }

        newFileHashCodeMap.putAll(currentFilesToLeaveAlone); // remember that these are still there

        if (!dryRun) {
            this.deploymentsMetadata.setCurrentDeployment(this.deploymentData.getDeploymentProps(), newFileHashCodeMap,
                true);
        }

        return newFileHashCodeMap;
    }

    /**
     * Create a zip file by adding all the files found in the file hashcode map. The
     * relative paths found in the map's key set are relative to the rootDir directory.
     * The files are stored in the given zipFile.
     * 
     * @param zipFile where to zip up all the files
     * @param rootDir all relative file paths are relative to this root directory
     * @param fileHashcodeMap the key set tells us all the files that need to be zipped up
     *
     * @throws Exception 
     */
    private void createZipFile(File zipFile, File rootDir, FileHashcodeMap fileHashcodeMap) throws Exception {
        if (zipFile.getParentFile() != null) {
            zipFile.getParentFile().mkdirs();
        }

        Set<File> childrenOfRootToDelete = new HashSet<File>();

        ZipOutputStream zos = null;
        try {
            zos = new ZipOutputStream(new FileOutputStream(zipFile));

            for (String relativeFileToZip : fileHashcodeMap.keySet()) {
                File fileToZip = new File(rootDir, relativeFileToZip);
                FileInputStream fis = null;
                try {
                    fis = new FileInputStream(fileToZip);
                    ZipEntry zipEntry = new ZipEntry(relativeFileToZip);
                    zos.putNextEntry(zipEntry);
                    StreamUtil.copy(fis, zos, false);
                } finally {
                    if (fis != null) {
                        fis.close();
                    }

                    // Remember the location to delete it later. To speed up the deletes, we only remember
                    // the locations of the direct children under the rootDir. Since we know all zip content
                    // is located under those locations, we'll recursively delete everything under those
                    // rootDir children. We obviously can't delete everything under rootDir because there
                    // could be files in there not related to our zip file.
                    File childOfRoot = fileToZip;
                    while (childOfRoot != null) {
                        File parent = childOfRoot.getParentFile();
                        if (parent.equals(rootDir)) {
                            childrenOfRootToDelete.add(childOfRoot);
                            break;
                        } else {
                            childOfRoot = parent;
                        }
                    }
                }
            }

        } finally {
            if (zos != null) {
                zos.close();
            }
        }

        // we are done zipping up all files, delete all the individual files that are exploded that are now in the zip
        for (File childOfRootToDelete : childrenOfRootToDelete) {
            FileUtil.purge(childOfRootToDelete, true);
        }

        return;
    }

    /**
     * Performs in-memory-only calculations to determine where new files would go and what
     * their new hashcodes would be.
     * 
     * @return map of file/hashcodes if there were to be written to disk
     * @throws Exception
     */
    private FileHashcodeMap getNewDeploymentFileHashcodeMap() throws Exception {
        FileHashcodeMap fileHashcodeMap = new FileHashcodeMap();

        // perform in-memory extraction and calculate hashcodes for all zip files 
        InMemoryZipFileVisitor visitor;
        for (File zipFile : this.deploymentData.getZipFiles()) {
            debug("Extracting zip [", zipFile, "] in-memory to determine hashcodes for all entries");
            Pattern realizeRegex = null;
            if (this.deploymentData.getZipEntriesToRealizeRegex() != null) {
                realizeRegex = this.deploymentData.getZipEntriesToRealizeRegex().get(zipFile);
            }
            visitor = new InMemoryZipFileVisitor(realizeRegex, this.deploymentData.getTemplateEngine());
            ZipUtil.walkZipFile(zipFile, visitor);
            fileHashcodeMap.putAll(visitor.getFileHashcodeMap());
        }

        MessageDigestGenerator generator = new MessageDigestGenerator();

        // calculate hashcodes for all raw files, perform in-memory realization when necessary
        for (Map.Entry<File, File> rawFile : this.deploymentData.getRawFiles().entrySet()) {
            // determine where the original file is and where it would go if we were writing it to disk
            File currentLocationFile = rawFile.getKey();
            File newLocationFile = rawFile.getValue();
            String newLocationPath = rawFile.getValue().getPath();
            if (!newLocationFile.isAbsolute()) {
                newLocationFile = new File(this.deploymentData.getDestinationDir(), newLocationFile.getPath());
            }

            String hashcode;

            boolean realize = false;
            if (this.deploymentData.getRawFilesToRealize() != null) {
                realize = this.deploymentData.getRawFilesToRealize().contains(currentLocationFile);
            }

            if (realize) {
                debug("Realizing file [", currentLocationFile, "] in-memory to determine its hashcode");

                // this entry needs to be realized, do it now in-memory (we assume realizable files will not be large)
                // note: tempateEngine will never be null if we got here
                FileInputStream in = new FileInputStream(currentLocationFile);
                byte[] rawFileContent = StreamUtil.slurp(in);
                String content = this.deploymentData.getTemplateEngine().replaceTokens(new String(rawFileContent));

                // now calculate the hashcode of the realized content
                generator.add(content.getBytes());
                hashcode = generator.getDigestString();
            } else {
                debug("Streaming file [", currentLocationFile, "] in-memory to determine its hashcode");

                BufferedInputStream in = new BufferedInputStream(new FileInputStream(currentLocationFile));
                try {
                    hashcode = generator.calcDigestString(in);
                } finally {
                    in.close();
                }
            }

            // store where the file will be and what its hashcode would be
            if (rawFile.getValue().isAbsolute()) {
                fileHashcodeMap.put(newLocationFile.getAbsolutePath(), hashcode);
            } else {
                fileHashcodeMap.put(newLocationPath, hashcode);
            }
        }

        return fileHashcodeMap;
    }

    /**
     * Takes all backup files found in a previous deployment and restores those files to their
     * original locations.
     * 
     * This method is usually called after a {@link #deploy(DeployDifferences, boolean)} has been completed
     * because it would be at that time when the previous deployment's information has been persisted
     * and is available. Rarely will you ever want to restore backup files without first deploying
     * content.
     *
     * @param prevDeploymentId the previous deployment ID which contains the backup files 
     * @param map the map containing filenames/hashcodes that will be adjusted to reflect the restored file hashcodes
     * @param diff used to store information about the restored files
     * @param dryRun if <code>true</code>, don't really restore the files, but log the files that would
     *               have been restored; <code>false</code> means you really want to restore the files
     * @throws Exception
     */
    private void restoreBackupFiles(int prevDeploymentId, FileHashcodeMap map, DeployDifferences diff, boolean dryRun)
        throws Exception {

        // do the relative backup files first - these go into the destination directory
        File backupDir = this.deploymentsMetadata.getDeploymentBackupDirectory(prevDeploymentId);
        String backupDirBase = backupDir.getAbsolutePath();
        if (!backupDirBase.endsWith(File.separator)) {
            backupDirBase += File.separator; // make sure it ends with a file separator so our string manipulation works
        }
        File destDir = this.deploymentData.getDestinationDir();
        restoreBackupFilesRecursive(backupDir, backupDirBase, destDir, map, diff, dryRun);

        // now do the external backup files - these go into directories external to the destination directory
        // note that if we are on windows, we have to do special things due to the existence of multiple root
        // directories (i.e. drive letters).
        Map<String, File> extBackupDirs;
        extBackupDirs = this.deploymentsMetadata.getDeploymentExternalBackupDirectoriesForWindows(prevDeploymentId);
        if (extBackupDirs == null) {
            // we are on a non-windows platform; there is only one main root directory - "/"
            File extBackupDir = this.deploymentsMetadata.getDeploymentExternalBackupDirectory(prevDeploymentId);
            extBackupDirs = new HashMap<String, File>(1);
            extBackupDirs.put("/", extBackupDir);
        }

        for (Map.Entry<String, File> entry : extBackupDirs.entrySet()) {
            String rootDir = entry.getKey();
            File extBackupDir = entry.getValue();
            String extBackupDirBase = extBackupDir.getAbsolutePath();
            if (!extBackupDirBase.endsWith(File.separator)) {
                extBackupDirBase += File.separator;
            }
            restoreExternalBackupFilesRecursive(rootDir, extBackupDir, extBackupDirBase, map, diff, dryRun);
        }

        return;
    }

    private void restoreBackupFilesRecursive(File file, String base, File destDir, FileHashcodeMap map,
        DeployDifferences diff, boolean dryRun) throws Exception {

        File[] children = file.listFiles();
        if (children != null) {
            for (File child : children) {
                if (child.isDirectory()) {
                    restoreBackupFilesRecursive(child, base, destDir, map, diff, dryRun);
                } else {
                    String childRelativePath = child.getAbsolutePath().substring(base.length());
                    //if (this.deploymentData.isManageRootDir() || new File(childRelativePath).getParent() != null) {
                    File restoredFile = new File(destDir, childRelativePath);
                    debug("Restoring backup file [", child, "] to [", restoredFile, "]. dryRun=", dryRun);
                    if (!dryRun) {
                        restoredFile.getParentFile().mkdirs();
                        String hashcode = copyFileAndCalcHashcode(child, restoredFile);
                        map.put(childRelativePath, hashcode);
                    } else {
                        map.put(childRelativePath, MessageDigestGenerator.getDigestString(child));
                    }
                    diff.addRestoredFile(childRelativePath, child.getAbsolutePath());
                    //} else {
                    //    debug("Skipping the restoration of the backed up file [", childRelativePath,
                    //        "] since this deployment was told to not manage the root directory");
                    //}
                }
            }
        }
        return;
    }

    private void restoreExternalBackupFilesRecursive(String rootDir, File backupDir, String base, FileHashcodeMap map,
        DeployDifferences diff, boolean dryRun) throws Exception {

        File[] children = backupDir.listFiles();
        if (children != null) {
            for (File child : children) {
                if (child.isDirectory()) {
                    restoreExternalBackupFilesRecursive(rootDir, child, base, map, diff, dryRun);
                } else {
                    String childRelativePath = child.getAbsolutePath().substring(base.length());
                    File restoredFile = new File(rootDir, childRelativePath);
                    debug("Restoring backup file [", child, "] to external location [", restoredFile, "]. dryRun="
                        + dryRun);
                    if (!dryRun) {
                        restoredFile.getParentFile().mkdirs();
                        String hashcode = copyFileAndCalcHashcode(child, restoredFile);
                        map.put(restoredFile.getAbsolutePath(), hashcode);
                    } else {
                        map.put(restoredFile.getAbsolutePath(), MessageDigestGenerator.getDigestString(child));
                    }
                    diff.addRestoredFile(restoredFile.getAbsolutePath(), child.getAbsolutePath());
                }
            }
        }
        return;
    }

    private String copyFileAndCalcHashcode(File src, File dest) throws Exception {
        BufferedInputStream is = null;
        BufferedOutputStream os = null;
        try {
            is = new BufferedInputStream(new FileInputStream(src));
            os = new BufferedOutputStream(new FileOutputStream(dest));
            StreamCopyDigest copier = new StreamCopyDigest();
            return copier.copyAndCalculateHashcode(is, os);
        } finally {
            try {
                if (is != null)
                    is.close();
            } catch (Exception ignore) {
            }

            try {
                if (os != null)
                    os.close();
            } catch (Exception ignore) {
            }
        }
    }

    private void purgeFileOrDirectory(File fileOrDir, Set<String> skippedFiles, int level, boolean deleteIt) {
        // make sure we only purge deployment files, never the metadata directory or its files
        // we also want to leave all skipped files alone - don't delete those since they are unrelated to our deployment
        if (fileOrDir != null && !fileOrDir.getName().equals(DeploymentsMetadata.METADATA_DIR)) {
            if (fileOrDir.isDirectory()) {
                File[] doomedFiles = fileOrDir.listFiles();
                if (doomedFiles != null) {
                    for (File doomedFile : doomedFiles) {
                        // Do not purge any skipped files - we want to skip them.
                        // All our skipped files are always at the top root dir (level 0),
                        // so we can ignore the skipped set if we are at levels 1 or below since there are no skipped files down there
                        if (level != 0 || !skippedFiles.contains(doomedFile.getName())) {
                            purgeFileOrDirectory(doomedFile, skippedFiles, level + 1, true); // call this method recursively
                        }
                    }
                }
            }

            if (deleteIt) {
                fileOrDir.delete();
            }
        }

        return;
    }

    private void debug(Object... objs) {
        if (log.isDebugEnabled()) {
            String bundleName = this.deploymentData.getDeploymentProps().getBundleName();
            String bundleVersion = this.deploymentData.getDeploymentProps().getBundleVersion();
            int deploymentId = this.deploymentData.getDeploymentProps().getDeploymentId();
            StringBuilder str = new StringBuilder();
            str.append("Bundle [").append(bundleName).append(" v").append(bundleVersion).append(']');
            str.append("; Deployment [").append(deploymentId).append("]: ");
            for (Object o : objs) {
                str.append(o);
            }
            log.debug(str.toString());
        }
    }
}
