/*
 * Distributed as part of mchange-commons-java 0.2.11
 *
 * Copyright (C) 2015 Machinery For Change, Inc.
 *
 * Author: Steve Waldman <swaldman@mchange.com>
 *
 * This library is free software; you can redistribute it and/or modify
 * it under the terms of EITHER:
 *
 *     1) The GNU Lesser General Public License (LGPL), version 2.1, as 
 *        published by the Free Software Foundation
 *
 * OR
 *
 *     2) The Eclipse Public License (EPL), version 1.0
 *
 * You may choose which license to accept if you wish to redistribute
 * or modify this work. You may offer derivatives of this work
 * under the license you have chosen, or you may provide the same
 * choice of license which you have been offered here.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received copies of both LGPL v2.1 and EPL v1.0
 * along with this software; see the files LICENSE-EPL and LICENSE-LGPL.
 * If not, the text of these licenses are currently available at
 *
 * LGPL v2.1: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html
 *  EPL v1.0: http://www.eclipse.org/org/documents/epl-v10.php 
 * 
 */

package com.mchange.v2.async;

import java.util.*;
import com.mchange.v2.log.*;

import java.io.StringWriter;
import java.io.PrintWriter;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.Random;
import com.mchange.v2.io.IndentedWriter;
import com.mchange.v2.util.ResourceClosedException;

public final class ThreadPoolAsynchronousRunner implements AsynchronousRunner
{
    final static MLogger logger = MLog.getLogger( ThreadPoolAsynchronousRunner.class );

    final static int POLL_FOR_STOP_INTERVAL                       = 5000; //milliseconds

    final static int DFLT_DEADLOCK_DETECTOR_INTERVAL              = 10000; //milliseconds
    final static int DFLT_INTERRUPT_DELAY_AFTER_APPARENT_DEADLOCK = 60000; //milliseconds
    final static int DFLT_MAX_INDIVIDUAL_TASK_TIME                = 0;     //milliseconds, <= 0 means don't enforce a max task time

    final static int DFLT_MAX_EMERGENCY_THREADS                   = 10;

    final static long PURGE_EVERY = 500L;

    int deadlock_detector_interval;
    int interrupt_delay_after_apparent_deadlock;
    int max_individual_task_time;

    int        num_threads;
    boolean    daemon;
    HashSet    managed;
    HashSet    available;
    LinkedList pendingTasks;

    Random rnd = new Random();

    Timer myTimer;
    boolean should_cancel_timer;

    TimerTask deadlockDetector = new DeadlockDetector();
    TimerTask replacedThreadInterruptor = null;

    Map stoppedThreadsToStopDates = new HashMap();

    String threadLabel;

    private ThreadPoolAsynchronousRunner( int num_threads, 
					  boolean daemon, 
					  int max_individual_task_time,
					  int deadlock_detector_interval, 
					  int interrupt_delay_after_apparent_deadlock,
					  Timer myTimer,
					  boolean should_cancel_timer,
					  String threadLabel )
    {
        this.num_threads = num_threads;
        this.daemon = daemon;
        this.max_individual_task_time = max_individual_task_time;
        this.deadlock_detector_interval = deadlock_detector_interval;
        this.interrupt_delay_after_apparent_deadlock = interrupt_delay_after_apparent_deadlock;
        this.myTimer = myTimer;
        this.should_cancel_timer = should_cancel_timer;

	this.threadLabel = threadLabel;

        recreateThreadsAndTasks();

        myTimer.schedule( deadlockDetector, deadlock_detector_interval, deadlock_detector_interval );

    }

    private ThreadPoolAsynchronousRunner( int num_threads, 
					  boolean daemon, 
					  int max_individual_task_time,
					  int deadlock_detector_interval, 
					  int interrupt_delay_after_apparent_deadlock,
					  Timer myTimer,
					  boolean should_cancel_timer)
    {
	this( num_threads, 
	      daemon, 
	      max_individual_task_time,
	      deadlock_detector_interval, 
	      interrupt_delay_after_apparent_deadlock,
	      myTimer,
	      should_cancel_timer,
	      null );
    }

    public ThreadPoolAsynchronousRunner( int num_threads, 
					 boolean daemon, 
					 int max_individual_task_time,
					 int deadlock_detector_interval, 
					 int interrupt_delay_after_apparent_deadlock,
					 Timer myTimer,
					 String threadLabel )
    {
        this( num_threads, 
	      daemon, 
	      max_individual_task_time,
	      deadlock_detector_interval, 
	      interrupt_delay_after_apparent_deadlock,
	      myTimer, 
	      false,
	      threadLabel);
    }

    public ThreadPoolAsynchronousRunner( int num_threads, 
					 boolean daemon, 
					 int max_individual_task_time,
					 int deadlock_detector_interval, 
					 int interrupt_delay_after_apparent_deadlock,
					 Timer myTimer )
    {
        this( num_threads, 
	      daemon, 
	      max_individual_task_time,
	      deadlock_detector_interval, 
	      interrupt_delay_after_apparent_deadlock,
	      myTimer, 
	      false );
    }

    public ThreadPoolAsynchronousRunner( int num_threads, 
					 boolean daemon, 
					 int max_individual_task_time,
					 int deadlock_detector_interval, 
					 int interrupt_delay_after_apparent_deadlock,
					 String threadLabel )
    {
        this( num_threads, 
	      daemon, 
	      max_individual_task_time,
	      deadlock_detector_interval, 
	      interrupt_delay_after_apparent_deadlock,
	      new Timer( true ), 
	      true,
	      threadLabel );
    }

    public ThreadPoolAsynchronousRunner( int num_threads, 
					 boolean daemon, 
					 int max_individual_task_time,
					 int deadlock_detector_interval, 
					 int interrupt_delay_after_apparent_deadlock )
    {
        this( num_threads, 
	      daemon, 
	      max_individual_task_time,
	      deadlock_detector_interval, 
	      interrupt_delay_after_apparent_deadlock,
	      new Timer( true ), 
	      true );
    }

    public ThreadPoolAsynchronousRunner( int num_threads, boolean daemon, Timer sharedTimer, String threadLabel )
    { 
        this( num_threads, 
	      daemon, 
	      DFLT_MAX_INDIVIDUAL_TASK_TIME, 
	      DFLT_DEADLOCK_DETECTOR_INTERVAL, 
	      DFLT_INTERRUPT_DELAY_AFTER_APPARENT_DEADLOCK, 
	      sharedTimer, 
	      false,
	      threadLabel ); 
    }

    public ThreadPoolAsynchronousRunner( int num_threads, boolean daemon, Timer sharedTimer )
    { 
        this( num_threads, 
	      daemon, 
	      DFLT_MAX_INDIVIDUAL_TASK_TIME, 
	      DFLT_DEADLOCK_DETECTOR_INTERVAL, 
	      DFLT_INTERRUPT_DELAY_AFTER_APPARENT_DEADLOCK, 
	      sharedTimer, 
	      false ); 
    }

    public ThreadPoolAsynchronousRunner( int num_threads, boolean daemon )
    { 
        this( num_threads, 
	      daemon, 
	      DFLT_MAX_INDIVIDUAL_TASK_TIME, 
	      DFLT_DEADLOCK_DETECTOR_INTERVAL, 
	      DFLT_INTERRUPT_DELAY_AFTER_APPARENT_DEADLOCK, 
	      new Timer( true ), 
	      true ); 
    }

    public synchronized void postRunnable(Runnable r)
    {
        try
        {
            pendingTasks.add( r );
            this.notifyAll();
            
            if (Debug.DEBUG && logger.isLoggable(MLevel.FINEST))
                logger.log(MLevel.FINEST, this + ": Adding task to queue -- " + r);
        }
        catch ( NullPointerException e )
        {
            //e.printStackTrace();
            if ( Debug.DEBUG )
            {
                if ( logger.isLoggable( MLevel.FINE ) )
                    logger.log( MLevel.FINE, "NullPointerException while posting Runnable -- Probably we're closed.", e );
            }
            throw new ResourceClosedException("Attempted to use a ThreadPoolAsynchronousRunner in a closed or broken state.");
        }
    }

    public synchronized int getThreadCount()
    { return managed.size(); }

    public void close( boolean skip_remaining_tasks )
    {
        synchronized ( this )
        {
            if (managed == null) return;
            deadlockDetector.cancel();
            //replacedThreadInterruptor.cancel();
            if (should_cancel_timer)
                myTimer.cancel();
            myTimer = null;
            for (Iterator ii = managed.iterator(); ii.hasNext(); )
            { 
                PoolThread stopMe = (PoolThread) ii.next();
                stopMe.gentleStop();
                if (skip_remaining_tasks)
                    stopMe.interrupt();
            }
            managed = null;

            if (!skip_remaining_tasks)
            {
                for (Iterator ii = pendingTasks.iterator(); ii.hasNext(); )
                {
                    Runnable r = (Runnable) ii.next();
                    new Thread(r).start();
                    ii.remove();
                }
            }
            available = null;
            pendingTasks = null;
        }
    }

    public void close()
    { close( true ); }

    public synchronized int getActiveCount()
    { return managed.size() - available.size(); }

    public synchronized int getIdleCount()
    { return available.size(); }

    public synchronized int getPendingTaskCount()
    { return pendingTasks.size(); }

    public synchronized String getStatus()
    { 
        /*
	  StringBuffer sb = new StringBuffer( 512 );
	  sb.append( this.toString() );
	  sb.append( ' ' );
	  appendStatusString( sb );
	  return sb.toString();
         */

        return getMultiLineStatusString();
    }

    // done reflectively for jdk 1.3/1.4 compatability
    public synchronized String getStackTraces()
    { return getStackTraces(0); }

    // protected by ThreadPoolAsynchronousRunner.this' lock
    // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock
    private String getStackTraces(int initial_indent)
    {
	assert Thread.holdsLock( this );

        if (managed == null)
            return null;

        try
        {
            Method m = Thread.class.getMethod("getStackTrace", (Class[]) null); // cast to suppress inexact type warning

            StringWriter sw = new StringWriter(2048);
            IndentedWriter iw = new IndentedWriter( sw );
            for (int i = 0; i < initial_indent; ++i)
                iw.upIndent();
            for (Iterator ii = managed.iterator(); ii.hasNext(); )
            {
                Object poolThread = ii.next();
                Object[] stackTraces = (Object[]) m.invoke( poolThread, (Object[]) null ); // cast to suppress inexact type warning
		printStackTraces( iw, poolThread, stackTraces );
            }
            for (int i = 0; i < initial_indent; ++i)
                iw.downIndent();
            iw.flush(); // useless, but I feel better
            String out = sw.toString();
            iw.close(); // useless, but I feel better;
            return out;
        }
        catch (NoSuchMethodException e)
        {
            if ( logger.isLoggable( MLevel.FINE ) )
                logger.fine( this + ": stack traces unavailable because this is a pre-Java 1.5 VM.");
            return null;
        }
        catch (Exception e)
        {
            if ( logger.isLoggable( MLevel.FINE ) )
                logger.log( MLevel.FINE, this + ": An Exception occurred while trying to extract PoolThread stack traces.", e);
            return null;
        }
    }

    // no pre-synchronization required
    private String getJvmStackTraces(int initial_indent)
    {
        try
        {
            Method m = Thread.class.getMethod("getAllStackTraces", (Class[]) null); // cast to suppress inexact type warning
	    Map threadMap = (Map) m.invoke( null, (Object[]) null ); // cast to suppress inexact type warning

            StringWriter sw = new StringWriter(2048);
            IndentedWriter iw = new IndentedWriter( sw );
            for (int i = 0; i < initial_indent; ++i)
                iw.upIndent();
            for (Iterator ii = threadMap.entrySet().iterator(); ii.hasNext(); )
            {
		Map.Entry entry = (Map.Entry) ii.next();
                Object poolThread = entry.getKey();
                Object[] stackTraces = (Object[]) entry.getValue();
		printStackTraces( iw, poolThread, stackTraces );
            }
            for (int i = 0; i < initial_indent; ++i)
                iw.downIndent();
            iw.flush(); // useless, but I feel better
            String out = sw.toString();
            iw.close(); // useless, but I feel better;
            return out;
        }
        catch (NoSuchMethodException e)
        {
            if ( logger.isLoggable( MLevel.FINE ) )
                logger.fine( this + ": JVM stack traces unavailable because this is a pre-Java 1.5 VM.");
            return null;
        }
        catch (Exception e)
        {
            if ( logger.isLoggable( MLevel.FINE ) )
                logger.log( MLevel.FINE, this + ": An Exception occurred while trying to extract PoolThread stack traces.", e);
            return null;
        }
    }

    // no pre-synchronization required
    private void printStackTraces(IndentedWriter iw, Object thread, Object[] stackTraces) throws IOException
    {
	iw.println( thread );
	iw.upIndent();
	for (int i = 0, len = stackTraces.length; i < len; ++i)
	    iw.println( stackTraces[i] );
	iw.downIndent();
    }

    public synchronized String getMultiLineStatusString()
    { return this.getMultiLineStatusString(0); }

    // protected by ThreadPoolAsynchronousRunner.this' lock
    // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock
    private String getMultiLineStatusString(int initial_indent)
    {
        try
        {
            StringWriter sw = new StringWriter(2048);
            IndentedWriter iw = new IndentedWriter( sw );

            for (int i = 0; i < initial_indent; ++i)
                iw.upIndent();

            if (managed == null)
            {
                iw.print("[");
                iw.print( this );
                iw.println(" closed.]");
            }
            else
            {
                HashSet active = (HashSet) managed.clone();
                active.removeAll( available );

                iw.print("Managed Threads: ");
                iw.println( managed.size() );
                iw.print("Active Threads: ");
                iw.println( active.size() );
                iw.println("Active Tasks: ");
                iw.upIndent();
                for (Iterator ii = active.iterator(); ii.hasNext(); )
                {
                    PoolThread pt = (PoolThread) ii.next();
                    iw.println( pt.getCurrentTask() );
		    iw.upIndent();
                    iw.print( "on thread: ");
                    iw.println( pt.getName() );
		    iw.downIndent();
                }
                iw.downIndent();
                iw.println("Pending Tasks: ");
                iw.upIndent();
                for (int i = 0, len = pendingTasks.size(); i < len; ++i)
                    iw.println( pendingTasks.get( i ) );
                iw.downIndent();
            }

            for (int i = 0; i < initial_indent; ++i)
                iw.downIndent();
            iw.flush(); // useless, but I feel better
            String out = sw.toString();
            iw.close(); // useless, but I feel better;
            return out;
        }
        catch (IOException e)
        {
            if (logger.isLoggable( MLevel.WARNING ))
                logger.log( MLevel.WARNING, "Huh? An IOException when working with a StringWriter?!?", e);
            throw new RuntimeException("Huh? An IOException when working with a StringWriter?!? " + e);
        }
    }

    // protected by ThreadPoolAsynchronousRunner.this' lock
    // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock
    private void appendStatusString( StringBuffer sb )
    {
        if (managed == null)
            sb.append( "[closed]" );
        else
        {
            HashSet active = (HashSet) managed.clone();
            active.removeAll( available );
            sb.append("[num_managed_threads: ");
            sb.append( managed.size() );
            sb.append(", num_active: ");
            sb.append( active.size() );
            sb.append("; activeTasks: ");
            boolean first = true;
            for (Iterator ii = active.iterator(); ii.hasNext(); )
            {
                if (first)
                    first = false;
                else
                    sb.append(", ");
                PoolThread pt = (PoolThread) ii.next();
                sb.append( pt.getCurrentTask() );
                sb.append( " (");
                sb.append( pt.getName() );
                sb.append(')');
            }
            sb.append("; pendingTasks: ");
            for (int i = 0, len = pendingTasks.size(); i < len; ++i)
            {
                if (i != 0) sb.append(", ");
                sb.append( pendingTasks.get( i ) );
            }
            sb.append(']');
        }
    }

    // protected by ThreadPoolAsynchronousRunner.this' lock
    // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock (or is ctor)
    private void recreateThreadsAndTasks()
    {
        if ( this.managed != null)
        {
            Date aboutNow = new Date();
            for (Iterator ii = managed.iterator(); ii.hasNext(); )
            {
                PoolThread pt = (PoolThread) ii.next();
                pt.gentleStop();
                stoppedThreadsToStopDates.put( pt, aboutNow );
                ensureReplacedThreadsProcessing();
            }
        }

        this.managed = new HashSet();
        this.available = new HashSet();
        this.pendingTasks = new LinkedList();
        for (int i = 0; i < num_threads; ++i)
        {
            Thread t = new PoolThread(i, daemon);
            managed.add( t );
            available.add( t );
            t.start();
        }
    }

    // protected by ThreadPoolAsynchronousRunner.this' lock
    // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock
    private void processReplacedThreads()
    {
        long about_now = System.currentTimeMillis();
        for (Iterator ii = stoppedThreadsToStopDates.keySet().iterator(); ii.hasNext(); )
        {
            PoolThread pt = (PoolThread) ii.next();
            if (! pt.isAlive())
                ii.remove();
            else
            {
                Date d = (Date) stoppedThreadsToStopDates.get( pt );
                if ((about_now - d.getTime()) > interrupt_delay_after_apparent_deadlock)
                {
                    if (logger.isLoggable(MLevel.WARNING))
                        logger.log(MLevel.WARNING, 
                                        "Task " + pt.getCurrentTask() + " (in deadlocked PoolThread) failed to complete in maximum time " +
                                        interrupt_delay_after_apparent_deadlock + "ms. Trying interrupt().");
                    pt.interrupt();
                    ii.remove();
                }
                //else keep waiting...
            }
            if (stoppedThreadsToStopDates.isEmpty())
                stopReplacedThreadsProcessing();
        }
    }

    // protected by ThreadPoolAsynchronousRunner.this' lock
    // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock
    private void ensureReplacedThreadsProcessing()
    {
        if (replacedThreadInterruptor == null)
        {
            if (logger.isLoggable( MLevel.FINE ))
                logger.fine("Apparently some threads have been replaced. Replacement thread processing enabled.");

            this.replacedThreadInterruptor = new ReplacedThreadInterruptor();
            int replacedThreadProcessDelay = interrupt_delay_after_apparent_deadlock / 4;
            myTimer.schedule( replacedThreadInterruptor, replacedThreadProcessDelay, replacedThreadProcessDelay );
        }
    }

    // protected by ThreadPoolAsynchronousRunner.this' lock
    // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock
    private void stopReplacedThreadsProcessing()
    {
        if (this.replacedThreadInterruptor != null)
        {
            this.replacedThreadInterruptor.cancel();
            this.replacedThreadInterruptor = null;

            if (logger.isLoggable( MLevel.FINE ))
                logger.fine("Apparently all replaced threads have either completed their tasks or been interrupted(). " +
                "Replacement thread processing cancelled.");
        }
    }

    // protected by ThreadPoolAsynchronousRunner.this' lock
    // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock
    private void shuttingDown( PoolThread pt )
    {
        if (managed != null && managed.contains( pt )) //we are not closed, and this was a thread in the current pool, not a replaced thread
        {
            managed.remove( pt );
            available.remove( pt );
            PoolThread replacement = new PoolThread( pt.getIndex(), daemon );
            managed.add( replacement );
            available.add( replacement );
            replacement.start();
        }
    }


    class PoolThread extends Thread
    {
        // protected by ThreadPoolAsynchronousRunner.this' lock
        Runnable currentTask;

        // protected by ThreadPoolAsynchronousRunner.this' lock
        boolean should_stop;

        // post ctor immutable
        int index;

        // not shared. only accessed by the PoolThread itself
        TimerTask maxIndividualTaskTimeEnforcer = null;

        PoolThread(int index, boolean daemon)
        {
            this.setName( (threadLabel == null ? this.getClass().getName() : threadLabel) + "-#" + index);
            this.setDaemon( daemon );
            this.index = index;
        }

        public int getIndex()
        { return index; }

        // protected by ThreadPoolAsynchronousRunner.this' lock
        // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock
        void gentleStop()
        { should_stop = true; }

        // protected by ThreadPoolAsynchronousRunner.this' lock
        // BE SURE CALLER OWNS ThreadPoolAsynchronousRunner.this' lock
        Runnable getCurrentTask()
        { return currentTask; }

        // no need to sync. data not shared
        private /* synchronized */  void setMaxIndividualTaskTimeEnforcer()
        {
            this.maxIndividualTaskTimeEnforcer = new MaxIndividualTaskTimeEnforcer( this );
            myTimer.schedule( maxIndividualTaskTimeEnforcer, max_individual_task_time );
        }

        // no need to sync. data not shared
        private /* synchronized */ void cancelMaxIndividualTaskTimeEnforcer()
        {
            this.maxIndividualTaskTimeEnforcer.cancel();
            this.maxIndividualTaskTimeEnforcer = null;
        }

        // no need to sync. Timer threadsafe, no other data access
        private void purgeTimer()
        { 
	    myTimer.purge(); 
	    if ( logger.isLoggable( MLevel.FINER ) )
		logger.log(MLevel.FINER, this.getClass().getName() + " -- PURGING TIMER");
	}

        public void run()
        {
	    long checkForPurge = rnd.nextLong();

            try
            {
                thread_loop:
                    while (true)
                    {
                        Runnable myTask;
                        synchronized ( ThreadPoolAsynchronousRunner.this )
                        {
                            while ( !should_stop && pendingTasks.size() == 0 )
                                ThreadPoolAsynchronousRunner.this.wait( POLL_FOR_STOP_INTERVAL );
                            if (should_stop) 
                                break thread_loop;

                            if (! available.remove( this ) )
                                throw new InternalError("An unavailable PoolThread tried to check itself out!!!");
                            myTask = (Runnable) pendingTasks.remove(0);
                            currentTask = myTask;
                        }
                        try
                        { 
                            if (max_individual_task_time > 0)
                                setMaxIndividualTaskTimeEnforcer();
                            myTask.run(); 
                        }
                        catch ( RuntimeException e )
                        {
                            if ( logger.isLoggable( MLevel.WARNING ) )
                                logger.log(MLevel.WARNING, this + " -- caught unexpected Exception while executing posted task.", e);
                            //e.printStackTrace();
                        }
                        finally
                        {
                            if ( maxIndividualTaskTimeEnforcer != null )
				{
				    cancelMaxIndividualTaskTimeEnforcer();

				    // we stochastically purge the timer roughly every PURGE_EVERY cancels
				    // math below is an inline, fast, pseudorandom long generator.
				    // see com.mchange.v2.util.XORShiftRandomUtils
				    checkForPurge ^= (checkForPurge << 21);
				    checkForPurge ^= (checkForPurge >>> 35);
				    checkForPurge ^= (checkForPurge << 4);
				    if ( (checkForPurge % PURGE_EVERY ) == 0 )
					purgeTimer();
				}

                            synchronized ( ThreadPoolAsynchronousRunner.this )
                            {
                                if (should_stop)
                                    break thread_loop;

                                if ( available != null && ! available.add( this ) )
                                    throw new InternalError("An apparently available PoolThread tried to check itself in!!!");
                                currentTask = null;
                            }
                        }
                    }
            }
            catch ( InterruptedException exc )
            {
//              if ( Debug.TRACE > Debug.TRACE_NONE )
//              System.err.println(this + " interrupted. Shutting down.");

                if ( Debug.TRACE > Debug.TRACE_NONE && logger.isLoggable( MLevel.FINE ) )
                    logger.fine(this + " interrupted. Shutting down.");
            }
            catch (RuntimeException re)
            {
                if (logger.isLoggable(MLevel.WARNING))
                    logger.log(MLevel.WARNING, "An unexpected RuntimException is implicated in the closing of " + this, re);
                throw re;
            }
            catch (Error err)
            {
                if (logger.isLoggable(MLevel.WARNING))
                    logger.log(MLevel.WARNING, 
                               "An Error forced the closing of " + this + 
                               ". Will attempt to reconstruct, but this might mean that something bad is happening.", 
                               err);
                throw err;
            }
            finally
            {
                synchronized ( ThreadPoolAsynchronousRunner.this )
                { ThreadPoolAsynchronousRunner.this.shuttingDown( this ); }
            }
        }
    }

    class DeadlockDetector extends TimerTask
    {
        LinkedList last = null;
        LinkedList current = null;

        public void run()
        {

            boolean run_stray_tasks = false;
            synchronized ( ThreadPoolAsynchronousRunner.this )
            { 
                if (pendingTasks.size() == 0)
                {
                    last = null;
		    if ( logger.isLoggable( MLevel.FINEST ) )
			logger.log( MLevel.FINEST, this + " -- Running DeadlockDetector[Exiting. No pending tasks.]");
                    return;
                }

                current = (LinkedList) pendingTasks.clone();
		if ( logger.isLoggable( MLevel.FINEST ) )
		    logger.log( MLevel.FINEST, this + " -- Running DeadlockDetector[last->" + last + ",current->" + current + ']');

                if ( current.equals( last ) )
                {
                    //System.err.println(this + " -- APPARENT DEADLOCK!!! Creating emergency threads for unassigned pending tasks!");
                    if ( logger.isLoggable( MLevel.WARNING ) )
                    {
                        logger.warning(this + " -- APPARENT DEADLOCK!!! Creating emergency threads for unassigned pending tasks!");
                        StringWriter sw = new StringWriter( 4096 );
                        PrintWriter pw = new PrintWriter( sw );
                        //StringBuffer sb = new StringBuffer( 512 );
                        //appendStatusString( sb );
                        //System.err.println( sb.toString() );
                        pw.print( this );
                        pw.println( " -- APPARENT DEADLOCK!!! Complete Status: ");
                        pw.print( ThreadPoolAsynchronousRunner.this.getMultiLineStatusString( 1 ) );
                        pw.println("Pool thread stack traces:"); 
                        String stackTraces = getStackTraces( 1 );
                        if (stackTraces == null)
                            pw.println("\t[Stack traces of deadlocked task threads not available.]");
                        else
                            pw.print( stackTraces ); //already has an end-of-line
                        pw.flush(); //superfluous, but I feel better
                        logger.warning( sw.toString() );
                        pw.close(); //superfluous, but I feel better
                    }
		    if ( logger.isLoggable( MLevel.FINEST ) )
		    {
                        StringWriter sw = new StringWriter( 4096 );
                        PrintWriter pw = new PrintWriter( sw );
                        pw.print( this );
                        pw.println( " -- APPARENT DEADLOCK extra info, full JVM thread dump: ");
                        String stackTraces = getJvmStackTraces( 1 );
                        if (stackTraces == null)
                            pw.println("\t[Full JVM thread dump not available.]");
                        else
                            pw.print( stackTraces ); //already has an end-of-line
                        pw.flush(); //superfluous, but I feel better
                        logger.finest( sw.toString() );
                        pw.close(); //superfluous, but I feel better
		    }
                    recreateThreadsAndTasks();
                    run_stray_tasks = true;
                }
            }
            if (run_stray_tasks)
            {
                AsynchronousRunner ar = new ThreadPerTaskAsynchronousRunner( DFLT_MAX_EMERGENCY_THREADS, max_individual_task_time );
                for ( Iterator ii = current.iterator(); ii.hasNext(); )
                    ar.postRunnable( (Runnable) ii.next() );
                ar.close( false ); //tell the emergency runner to close itself when its tasks are complete
                last = null;
            }
            else
                last = current;

            // under some circumstances, these lists seem to hold onto a lot of memory... presumably this
            // is when long pending task lists build up for some reason... nevertheless, let's dereference
            // things as soon as possible. [Thanks to Venkatesh Seetharamaiah for calling attention to this
            // issue, and for documenting the source of object retention.]
            current = null;
        }
    }

    class MaxIndividualTaskTimeEnforcer extends TimerTask
    {
        PoolThread pt;
        Thread     interruptMe;
        String     threadStr;
        String     fixedTaskStr;

        MaxIndividualTaskTimeEnforcer(PoolThread pt)
        { 
            this.pt = pt;
            this.interruptMe = pt;
            this.threadStr = pt.toString();
            this.fixedTaskStr = null;
        }

        MaxIndividualTaskTimeEnforcer(Thread interruptMe, String threadStr, String fixedTaskStr)
        { 
            this.pt = null; 
            this.interruptMe = interruptMe;
            this.threadStr = threadStr;
            this.fixedTaskStr = fixedTaskStr;
        }

        public void run() 
        { 
            String taskStr;

            if (fixedTaskStr != null)
                taskStr = fixedTaskStr;
            else if (pt != null)
            {
                synchronized (ThreadPoolAsynchronousRunner.this)
                { taskStr = String.valueOf( pt.getCurrentTask() ); }
            }
            else
                taskStr = "Unknown task?!";

            if (logger.isLoggable( MLevel.WARNING ))
                logger.warning("A task has exceeded the maximum allowable task time. Will interrupt() thread [" + threadStr
                                + "], with current task: " + taskStr);

            interruptMe.interrupt(); 

            if (logger.isLoggable( MLevel.WARNING ))
                logger.warning("Thread [" + threadStr + "] interrupted.");
        } 
    }

    //not currently used...
    private void runInEmergencyThread( final Runnable r )
    {
        final Thread t = new Thread( r );
        t.start();
        if (max_individual_task_time > 0)
        {
            TimerTask maxIndividualTaskTimeEnforcer = new MaxIndividualTaskTimeEnforcer(t, t + " [One-off emergency thread!!!]", r.toString());
            myTimer.schedule( maxIndividualTaskTimeEnforcer, max_individual_task_time );
        }
    }

    class ReplacedThreadInterruptor extends TimerTask
    {
        public void run()
        {
            synchronized (ThreadPoolAsynchronousRunner.this)
            { processReplacedThreads(); }
        }
    }
}
