/* Copyright (C) 2003-2008 The University of Iowa 
 *
 * This file is part of the Das2 <www.das2.org> utilities library.
 *
 * Das2 utilities are free software: you can redistribute and/or modify them
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * Das2 utilities are distributed in the hope that they will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 * License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * as well as the GNU General Public License along with Das2 utilities.  If
 * not, see <http://www.gnu.org/licenses/>.
 *
 * WebFileSystem.java
 *
 * Created on May 13, 2004, 1:22 PM
 *
 * A WebFileSystem allows web files to be opened just as if they were
 * local files, since it manages the transfer of the file to a local
 * file system.
 */
package org.das2.util.filesystem;

import org.das2.util.monitor.CancelledOperationException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import org.das2.util.Base64;
import org.das2.util.monitor.ProgressMonitor;
import org.das2.util.filesystem.FileSystem.FileSystemOfflineException;
import java.util.concurrent.locks.Lock;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import org.das2.util.FileUtil;
import org.das2.util.OsUtil;
import static org.das2.util.filesystem.FileSystem.toCanonicalFilename;
import org.das2.util.monitor.NullProgressMonitor;

/**
 * Make a web folder accessible.  This assumes listings are provided in html form by requesting links ending in a slash.
 * For example, http://autoplot.org/data/pngwalk/.  Links to resources "outside" of the filesystem are not considered part of
 * the filesystem.  Again, this assumes listings are HTML content, and I suspect this will be changing (xml+client-side-xslt)...
 *
 * @author  Jeremy
 */
public class HttpFileSystem extends WebFileSystem {

    protected static final Logger logger= org.das2.util.LoggerManager.getLogger( "das2.filesystem.http" );

    /**
     * provide some caching for directory entries.
     */
    private final Map<String,DirectoryEntry> listingEntries= new HashMap();
    private final Map<String,Long> listingEntryFreshness= new HashMap();
    
    /** 
     * Create a new HttpFileSystem mirroring the root, a URL pointing to "http" or "https", 
     * in the local folder.
     * @param root the root of the filesystem
     * @param localRoot the local root where files are downloaded.
     */
    protected HttpFileSystem(URI root, File localRoot) {
        super(root, localRoot);
    }
    
    private String cookie=null;
    
    /**
     * return the cookie needed.
     * @return 
     */
    protected String getCookie() {
        return this.cookie;
    }
    
    /**
     * Create a filesystem from the URI.  Note "user@" will be added to the 
     * URI if credentials are needed and added automatically.
     * @param rooturi
     * @return the filesystem.
     * @throws org.das2.util.filesystem.FileSystem.FileSystemOfflineException
     * @throws UnknownHostException
     * @throws FileNotFoundException 
     */
    public static HttpFileSystem createHttpFileSystem(URI rooturi) throws FileSystemOfflineException, UnknownHostException, FileNotFoundException {
        try {
            
            String auth= rooturi.getAuthority();
            if ( auth==null ) {
                throw new MalformedURLException("URL does not contain authority, check for ///");
            }
            
            if ( rooturi.toString().contains("$Y") ) {
                logger.fine( "somehow template leaked into FileSystem code.");
            }
            
            String[] ss= auth.split("@");

            URL root;

            if ( ss.length>3 ) {
                throw new IllegalArgumentException("user info section can contain at most two at (@) symbols");
            } else if ( ss.length==3 ) {//bugfix 3299977.  UMich server uses email:password@umich.  Java doesn't like this.
                // the user didn't escape the at (@) in the email.  escape it here.
                StringBuilder userInfo= new StringBuilder( ss[0] );
                for ( int i=1;i<2;i++ ) userInfo.append("%40").append(ss[i]);
                auth= ss[2];
                try {
                    URI rooturi2= new URI( rooturi.getScheme() + "://" + userInfo.toString()+"@"+auth + rooturi.getPath() );
                    rooturi= rooturi2;
                } catch ( URISyntaxException ex ) {
                    throw new IllegalArgumentException("unable to handle: "+rooturi);
                }
            }

            root= rooturi.toURL();
            
            logger.log(Level.FINER, "See https://www.draw.io/#G0B1Ywc5_Vexx1d3ctdGZxZDNkM3M" );
            logger.log(Level.FINER, "URL Reference: {0}", root);

            boolean doCheck= true;
            URI parentURI= FileSystemUtil.getParentUri( rooturi );
            if ( parentURI!=null ) {
                HttpFileSystem parent= (HttpFileSystem) peek( parentURI );
                if ( parent!=null && parent.isOffline() ) {
                    logger.finer("parent is offline, do not check...");
                    doCheck= false;
                }
            }


            boolean offline = true;
            String offlineMessage= "";
            int offlineResponseCode= 0;
            
            String cookie= null;
            
            while ( doCheck && !FileSystem.settings().isOffline() ) {
                
                // verify URL is valid and accessible
                HttpURLConnection urlc = (HttpURLConnection) root.openConnection();
                
                urlc.setConnectTimeout( FileSystem.settings().getConnectTimeoutMs() );
                urlc.setReadTimeout( FileSystem.settings().getReadTimeoutMs() );
                //urlc.setRequestMethod("HEAD"); // Causes problems with the LANL firewall.

                String userInfo;  // null means that userInfo has not been attempted.

                try {
                    logger.log(Level.FINER, "Check keychain: ", root);
                    userInfo = KeyChain.getDefault().getUserInfo(root);
                } catch (CancelledOperationException ex) {
                    logger.log( Level.FINER, "user cancelled credentials for {0}", rooturi);
                    break;
                }
                
                if ( userInfo != null) {
                    String encode = Base64.getEncoder().encodeToString( userInfo.getBytes());
                    urlc.setRequestProperty("Authorization", "Basic " + encode);
                }

                cookie= KeyChain.getDefault().getCookie(root);
                if ( cookie!=null ) {
                    urlc.setRequestProperty("Cookie",cookie);
                }
                
                int responseCode= -1;
                try {
                    logger.log( Level.FINER, "Verify Credentials {0}", urlc );
                    if ( userInfo!=null && !userInfo.contains(":") ) {
                        logger.log( Level.INFO, "urlc={0}", urlc );
                        logger.log( Level.INFO, "userInfo does not appear to contain password: {0}", userInfo );
                    } else {
                        logger.log( Level.FINER, "userInfo.length={0}", ( userInfo==null ? -1 : userInfo.length() ));
                    }
                    urlc.connect();
                    responseCode= urlc.getResponseCode();
                    
                    logger.log( Level.FINER, "made connection, now consume rest of stream: {0}", urlc );
                    try {
                        HttpUtil.consumeStream( urlc.getInputStream() );
                    } catch ( IOException ex ) {
                        logger.fine("exception when politely consuming stream after initial check");
                    }
                    logger.log( Level.FINER, "done consuming and initial connection is complete: {0}" );
                    urlc.disconnect();
                    offline= false;
                    doCheck= false;
                    logger.finer( "Verify Credentials exits with okay");
                    
                } catch ( SocketTimeoutException ex ) {
                    logger.finer("Socket timeout");
                    HttpUtil.consumeStream( urlc.getErrorStream() );
                    responseCode= HttpURLConnection.HTTP_GATEWAY_TIMEOUT;
                    offlineMessage= "socket timeout";
                    offlineResponseCode= HttpURLConnection.HTTP_GATEWAY_TIMEOUT;
                    
                    doCheck= false;
                        
                } catch ( IOException ex ) {
                    
                    logger.finer("Error with credentials");
                    int code= 0;
                    String msg;
                    try {
                        code= urlc.getResponseCode();
                        msg= urlc.getResponseMessage();
                    } catch ( IOException ex2 ) {
                        // do nothing in this case, just try to get a response code.
                        logger.log(Level.SEVERE,ex2.getMessage(),ex2);
                        msg= ex2.getMessage();
                    }
                    
                    HttpUtil.consumeStream( urlc.getErrorStream() );
                    
                    if ( code==HttpURLConnection.HTTP_NOT_FOUND ) {
                        logger.log( Level.SEVERE, String.format( "%d: folder not found: %s\n%s", code, root, msg ), ex );
                        throw (FileNotFoundException)ex;
                    } else if ( code==HttpURLConnection.HTTP_BAD_REQUEST ) { // bad request--used by raw.githubusercontent.com/
                        // we will use this as a flag to indicate a file could be downloaded.
                        offlineMessage= "listing results in bad request";
                        logger.log( Level.SEVERE, String.format( "%d: folder cannot be listed: %s\n%s", code, root, msg ), ex );
                        doCheck= false;
                        
                    } else if ( code!=HttpURLConnection.HTTP_UNAUTHORIZED ) {
                        // Note this may still be code 403.  We still enter the same branch for now, because the user might be on a network that isn't permitted now.
                        logger.log( Level.SEVERE, String.format( "%d: failed to connect to %s\n%s", code, root, msg ), ex );
                        if ( FileSystem.settings().isAllowOffline() ) {
                            logger.info("remote filesystem is offline, allowing access to local cache.");
                            break;
                        } else {
                            throw new FileSystemOfflineException("" + code + ": " + msg );
                        }
                    }
                    
                    if ( "true".equals( System.getProperty("java.awt.headless") ) && userInfo!=null ) { 
                        logger.finer( "Headless mode means we have to give up");
                        if ( FileSystem.settings().isAllowOffline() ) {
                            logger.info("remote filesystem is offline, allowing access to local cache.");
                            break;
                        } else {
                            throw new FileSystemOfflineException("" + code + ": " + msg );
                        }
                    }
                    
                    if ( offlineMessage.length()==0 ) offlineMessage= msg;
                    offlineResponseCode= code;
                }

                if ( responseCode==HttpURLConnection.HTTP_NOT_FOUND ) {
                    throw new FileNotFoundException("root returns 404, indicating it does not exist");
                }
                
                if ( responseCode!=HttpURLConnection.HTTP_GATEWAY_TIMEOUT ) {
                    if ( responseCode != HttpURLConnection.HTTP_OK && responseCode != HttpURLConnection.HTTP_FORBIDDEN) {
                        if ( responseCode==HttpURLConnection.HTTP_UNAUTHORIZED ) {
                            // might be nice to modify URL so that credentials are used.
                            if ( userInfo!=null ) {
                                KeyChain.getDefault().clearUserPassword(root);
                            }
                            if ( userInfo==null ) {
                                String port=  root.getPort()==-1 ? "" : ( ":" +root.getPort() );
                                URL rootAuth= new URL( root.getProtocol() + "://" + "user@" + root.getHost() + port + root.getFile() );
                                try {
                                    URI rootAuthUri= rootAuth.toURI();
                                    rooturi= rootAuthUri;
                                    root= rooturi.toURL();

                                } catch ( URISyntaxException ex ) {
                                    throw new RuntimeException(ex);
                                }
                            }
                        } else if ( responseCode==HttpURLConnection.HTTP_BAD_REQUEST ) {
                            offline= true; // we can get files we know about already, but listings cannot be done.
                        } else {
                            offline= false;
                        }
                    } else {
                        offline= false;
                    }
                }

            }
            
            File local;

            if (FileSystemSettings.hasAllPermission()) {
                local = localRoot(rooturi);
                logger.log(Level.FINER, "initializing httpfs {0} at {1}", new Object[]{root, local});
            } else {
                local = null;
                logger.log(Level.FINER, "initializing httpfs {0} in applet mode", root);
            }
            HttpFileSystem result = new HttpFileSystem(rooturi, local);

            if ( offline ) {
                logger.log( Level.WARNING, "filesystem is offline: {0}", rooturi );
            }
            
            result.offline = offline;
            result.offlineMessage= offlineMessage;
            result.offlineResponseCode= offlineResponseCode;
            result.cookie= cookie;

            return result;

        } catch (FileSystemOfflineException | FileNotFoundException | UnknownHostException e) {
            throw e;
        } catch (IOException e) {
            throw new FileSystemOfflineException(e,rooturi);
        }

    }

    /**
     * It looks like an external process (another das2 app) is downloading the resource.  Wait for the other
     * process to download the file.  If the file is idle for more than the allowable external idle millisecond limit
     * (FileSystemSettings.allowableExternalIdleMs), then return false.
     * @param f the file
     * @param partFile the part file we're watching
     * @param monitor 
     * @return true if the other app appears to have loaded the resource, false otherwise.
     */
    private boolean waitDownloadExternal( File f, File partFile, ProgressMonitor monitor ) {
        monitor.setProgressMessage("waiting for other process load");
        while ( partFile.exists() && ( System.currentTimeMillis() - partFile.lastModified() ) < FileSystemSettings.allowableExternalIdleMs ) {
            try {
                Thread.sleep(300);
                logger.log(Level.FINEST, "waiting for external process to download {0}", partFile);
                monitor.setTaskProgress(partFile.length());
                if ( monitor.isCancelled() ) {
                    return false;
                }
            } catch (InterruptedException ex) {
                logger.log(Level.SEVERE, ex.getMessage(), ex);
            }
        }
        if ( partFile.exists() ) {
            logger.finer("timeout waiting for partFile to be deleted");
            return false;
        } else {
            if ( f.exists() ) {
                logger.finer("successfully waited for external download to complete");
                return true;
            } else {
                logger.finer("part file removed but complete file is not found");
                return false;
            }
        }
    }
    
    /**
     * pull out some of the headers to record ETag and Content_Type.
     * @param connect
     * @return 
     */
    protected Map<String,String> reduceMeta( URLConnection connect ) {
        Map<String,String> result= new HashMap<>();
        result.put( WebProtocol.META_ETAG, connect.getHeaderField( WebProtocol.META_ETAG ) );
        result.put( WebProtocol.META_CONTENT_TYPE, connect.getHeaderField( WebProtocol.META_CONTENT_TYPE ) );
        if ( connect instanceof HttpURLConnection ) {
            try {
                result.put( WebProtocol.HTTP_RESPONSE_CODE, String.valueOf(((HttpURLConnection)connect).getResponseCode()) );
            } catch ( IOException ex ) {
                // do nothing as before.
            }
        }
        return result;
    }
    
    private Map<String,String> doTryDownload( URLConnection urlc, 
            URL remoteURL, 
            String filename,
            File f, 
            File partFile, 
            ProgressMonitor monitor ) throws IOException {
        
        Map<String,String> result;
        InputStream in;

        if ( loggerUrl.isLoggable(Level.FINE) && urlc.getURL().getPath().endsWith("/") ) {
            loggerUrl.log(Level.FINE, "GET to get listing {0}", new Object[] { urlc.getURL() } );
        } else {
            loggerUrl.log(Level.FINE, "GET to get data {0}", new Object[] { urlc.getURL() } );
        }

        in= urlc.getInputStream();

        HttpURLConnection hurlc = (HttpURLConnection) urlc;
        if (hurlc.getResponseCode() == 404) {
            logger.log(Level.INFO, "{0} URL: {1}", new Object[]{hurlc.getResponseCode(), remoteURL});
            throw new FileNotFoundException("not found: " + remoteURL);
        } else if (hurlc.getResponseCode() != 200) {
            logger.log(Level.INFO, "{0} URL: {1}", new Object[]{hurlc.getResponseCode(), remoteURL});
            throw new IOException( hurlc.getResponseCode()+": "+ hurlc.getResponseMessage() + "\n"+remoteURL );
        }

        Date d;
        List<String> sd= urlc.getHeaderFields().get("Last-Modified");
        if ( sd!=null && sd.size()>0 ) {
            d= new Date( sd.get(sd.size()-1) );
        } else {
            d= new Date();
        }

        long expectedContentLength= urlc.getContentLengthLong();
        monitor.setTaskSize( expectedContentLength );

        if (!f.getParentFile().exists()) {
            logger.log(Level.FINER, "make dirs {0}", f.getParentFile());
            FileSystemUtil.maybeMkdirs( f.getParentFile() );
        }
        if (partFile.exists()) {
            logger.log(Level.FINER, "partFile exists {0}", partFile);
            long ageMillis= System.currentTimeMillis() - partFile.lastModified(); // TODO: this is where OS-level locking would be nice...
            if ( ageMillis<FileSystemSettings.allowableExternalIdleMs ) { // if it's been modified less than sixty seconds ago, then wait to see if it goes away, then check again.
                if ( waitDownloadExternal( f, partFile, monitor ) ) {
                    return Collections.EMPTY_MAP; // success
                } else {
                    if ( monitor.isCancelled() ) {
                        throw new InterruptedIOException("interrupt while waiting for external process to download "+partFile);
                    } else {
                        throw new IOException( "timeout waiting for external process to download "+partFile );
                    }
                }
            } else {
                if (!partFile.delete()) {
                    logger.log(Level.INFO, "Unable to delete part file {0}, using new name for part file.", partFile ); //TODO: review this
                    partFile= new File( f.toString()+".part."+System.currentTimeMillis() );
                }
            }
        }

        if (partFile.createNewFile()) {
            //InputStream in;
            //in = urlc.getInputStream();

            logger.log(Level.FINER, "transferring bytes of {0}", filename);
            FileOutputStream out = new FileOutputStream(partFile);
            monitor.setLabel("downloading file");
            monitor.started();
            try {
                // https://sourceforge.net/p/autoplot/bugs/1229/
                String contentLocation= urlc.getHeaderField("Content-Location");
                String contentType= urlc.getHeaderField("Content-Type");
                result= reduceMeta( urlc );

                boolean doUnzip= !filename.endsWith(".gz" ) && "application/x-gzip".equals( contentType ) && ( contentLocation==null || contentLocation.endsWith(".gz") );
                if ( doUnzip ) {
                    in= new GZIPInputStream( in );
                }

                long totalBytesRead= copyStream(in, out, monitor);
                if ( totalBytesRead<urlc.getContentLength() ) {
                    logger.log(Level.WARNING, "fewer bytes downloaded than expected: {0} of {1}", new Object[]{totalBytesRead, expectedContentLength});
                    throw new IOException("fewer bytes in HTTP response than stated in header.");
                }
                monitor.finished();
                out.close();
                in.close();                    

                try {
                    partFile.setLastModified(d.getTime()+HTTP_CHECK_TIMESTAMP_LIMIT_MS);
                } catch ( Exception ex ) {
                    logger.log( Level.SEVERE, "unable to setLastModified", ex );
                }

                if ( f.exists() ) {
                    if ( f.isDirectory() ) {
                        logger.finer("file was once a directory.");
                        if ( !FileUtil.deleteFileTree(f) ) {
                            throw new IllegalArgumentException("unable to folder to make way for file: "+f );
                        }
                    } else {
                        if ( f.length()==partFile.length() ) {
                            if ( OsUtil.contentEquals(f, partFile ) ) {
                                logger.finer("another thread must have downloaded file.");
                                if ( f.lastModified()==0 ) {
                                    logger.finer("existing file didn't have a proper timetag, copy timetag from part file.");
                                    if ( !f.setLastModified( partFile.lastModified() ) ) {
                                        logger.log(Level.INFO, "unable to set last modified on {0}", f);
                                    }
                                }
                                if ( !partFile.delete() ) {
                                    throw new IllegalArgumentException("unable to delete "+partFile );
                                }
                                return result;
                            } else {
                                logger.finer("another thread must have downloaded different file.");
                            }
                        }
                        logger.log(Level.FINER, "deleting old file {0}", f);
                        if ( !f.delete() ) {
                            throw new IllegalArgumentException("unable to delete "+f );
                        }
                    }
                }
                if ( !partFile.renameTo(f) ) {
                    logger.log(Level.WARNING, "rename failed {0} to {1}", new Object[]{partFile, f});
                    throw new IllegalArgumentException( "rename failed " + partFile + " to "+f );
                }
            } catch (IOException e) {
                out.close();
                in.close();
                logger.log( Level.FINER, "deleting partial download file {0}", partFile);
                if ( partFile.exists() && !partFile.delete() ) {
                    throw new IllegalArgumentException("unable to delete "+partFile );
                }
                throw e;
            }
        } else {
            throw new IOException("could not create local file: " + f);
        }
        return result;
    }
    
    /**
     * 
     * @param filename identifier for the resource, and should this end in gz, then the stream will probably be unzipped.
     * @param remoteURL the remote URL from which a connection is opened.
     * @param f the local file where the content will be stored.
     * @param partFile a temporary local file.
     * @param monitor a monitor for the download.
     * @return ETag if available
     * @throws IOException
     * @throws FileNotFoundException 
     */
    private Map<String,String> doDownload( String filename, URL remoteURL, File f, File partFile, ProgressMonitor monitor ) throws IOException, FileNotFoundException {
        
        logger.log(Level.FINE, "doDownload {0}", remoteURL);
        Map<String,String> result=null;
        
        loggerUrl.log(Level.FINE, "open connection to {0}", remoteURL);
        HttpURLConnection urlc = (HttpURLConnection)remoteURL.openConnection();
        urlc.setConnectTimeout( FileSystem.settings().getConnectTimeoutMs() );
        urlc.setReadTimeout( FileSystem.settings().getReadTimeoutMs() );

        //bug http://sourceforge.net/p/autoplot/bugs/1393/ shows where this is necessary.
        urlc.setUseCaches(false);
        
        String userInfo;
        try {
            userInfo = KeyChain.getDefault().getUserInfo(root);
        } catch (CancelledOperationException ex) {
            throw new IOException("user cancelled at credentials entry");
        }
        if ( userInfo != null) {
            String encode = Base64.getEncoder().encodeToString(userInfo.getBytes());
            urlc.setRequestProperty("Authorization", "Basic " + encode);
        }

        if ( cookie!=null ) {
            urlc.addRequestProperty("Cookie", cookie );
        }
        
        URLConnection oldurlc= urlc;
        urlc= (HttpURLConnection)HttpUtil.checkRedirect(urlc);

        if ( !urlc.equals(oldurlc) ) {
            userInfo= KeyChain.getDefault().checkUserInfo( urlc.getURL() );
            if ( userInfo!=null ) {
                String encode = Base64.getEncoder().encodeToString(userInfo.getBytes());
                try {
                    urlc.setRequestProperty("Authorization", "Basic " + encode);
                } catch ( IllegalStateException ex ) { // "Already connected"
                    logger.info("We are already connected, so resetting credentials would cause Already connected error");
                }
            }
        }

        try {
        
            boolean haveIt= false;
            while ( !haveIt ) {
                try {
                    result= doTryDownload( urlc, remoteURL, filename, f, partFile, monitor );
                    haveIt= true;
                    
                } catch ( IOException ex ) {
                    if ( ex.getCause()!=null && ex.getCause() instanceof CancelledOperationException ) {
                        throw ex;
                    }
                    if ( ((HttpURLConnection)urlc).getResponseCode()==401 ) {
                        if ( result==null ) result= new HashMap<>();
                        result.put( WebProtocol.HTTP_RESPONSE_CODE, String.valueOf( ((HttpURLConnection) urlc).getResponseCode() ) );
                        URL theUrl= urlc.getURL();
                        try {
                            KeyChain.getDefault().clearUserPassword( theUrl );
                            userInfo= KeyChain.getDefault().getUserInfo( theUrl, "user:pass" ); // mark this as a URL which needs credentials.
                        } catch (CancelledOperationException ex1) {
                            throw ex;
                        }
                        HttpURLConnection newConnection= (HttpURLConnection)theUrl.openConnection();
                        HttpUtil.copyConnectProperties( urlc, newConnection );
                        String encode = Base64.getEncoder().encodeToString(userInfo.getBytes());
                        newConnection.setRequestProperty("Authorization", "Basic " + encode);
                        urlc.disconnect();
                        urlc= newConnection;
                    } else {
                        throw ex;
                    }
                }
            }
        } finally {
            if ( urlc instanceof HttpURLConnection ) {
                if ( remoteURL.getPath().endsWith("/") ) {
                    logger.fine("not closing, because it was a listing file.");
                } else {
                    ((HttpURLConnection)urlc).disconnect();
                }
            }
        }
        return result;
    }
    /**
     * 
     * @param filename filename within the filesystem.
     * @param targetFile the target filename where the file is to be download.
     * @param partFile  use this file to stage the download
     * @param monitor  monitor the progress.
     * @return metadata containing ETag if available.
     * @throws IOException 
     */
    @Override
    protected Map<String,String> downloadFile(String filename, File targetFile, File partFile, ProgressMonitor monitor) throws IOException {
        
        Lock lock = getDownloadLock(filename, targetFile, monitor);
            
        if (lock == null) {
            return Collections.EMPTY_MAP;
        }
            
        Map<String,String> meta= Collections.EMPTY_MAP;
        
        filename = toCanonicalFilename(filename);
                
        logger.log(Level.FINER, "downloadFile {0}, using temporary file {1}", new Object[] { filename, partFile } );

        try {
            URL remoteURL = getURL( filename ); // TEMP see 650 above
            
            try { 
                meta= doDownload( filename, remoteURL, targetFile, partFile, monitor );
            } catch ( FileNotFoundException ex ) {
                if ( !filename.endsWith("/") ) {
                    remoteURL= new URL(root.toString() + filename.substring(1) + ".gz" );
                    try {
                        doDownload( filename, remoteURL, targetFile, partFile, monitor );
                    } catch ( FileNotFoundException exgz ) {
                        throw ex;
                    }
                }
            }

        } finally {

            lock.unlock();

        }
        return meta;
    }

    /**
     * this is introduced to support checking if the symbol foo/bar is a folder by checking
     * for a 303 redirect.
     *<blockquote><pre><small>{@code
     *   EXIST->Boolean
     *   REAL_NAME->String
     *}</small></pre></blockquote>    
     * others are just HTTP header fields like (see wget --server-response https://raw.githubusercontent.com/autoplot/jyds/master/dd.jyds):
     *<blockquote><pre><small>{@code
     *   Content-Length   the length in bytes of the resource.
     *   Cache-Control    max-age=300
     *   Date: Fri, 18 Jul 2014 12:07:18 GMT  time stamp.
     *   ETag: "750d4f66c58a0ac7fef2784253bf6954d4d38a85"
     *   Accept-Ranges    accepts requests for part of a file.
     *}</small></pre></blockquote>     
     * @param f the name within the filesystem
     * @return the metadata, such as the Date and ETag.
     * @throws java.io.IOException
     * @throws org.das2.util.monitor.CancelledOperationException
     */
    protected Map<String, Object> getHeadMeta(String f) throws IOException, CancelledOperationException {

        try {
            URL ur = new URL(this.root.toURL().toString() + ( f.length()>0 ? f.substring(1) : f ) );
            Map<String,String> meta= HttpUtil.getMetadata( ur, null );
            
            Map<String,Object> result= new HashMap<>();
            result.putAll(meta);
            
            result.put( "EXIST", Boolean.parseBoolean(meta.get( WebProtocol.META_EXIST ) ) );
            result.put( WebProtocol.META_EXIST, Boolean.parseBoolean(meta.get( WebProtocol.META_EXIST ) ) );  // exist is lower case.
            result.put( WebProtocol.META_CONTENT_LENGTH, Long.parseLong(meta.get(WebProtocol.META_CONTENT_LENGTH) ) );
            result.put( WebProtocol.META_LAST_MODIFIED, Long.parseLong(meta.get(WebProtocol.META_LAST_MODIFIED ) ) );

            return result;

        } catch (MalformedURLException ex) {
            throw new IllegalArgumentException(ex);
        }

    }

    /** dumb method looks for / in parent directory's listing.  Since we have
     * to list the parent, then IOException can be thrown.
     * 
     * @return true if the name appears to be a directory (folder).
     * @throws java.io.IOException
     */
    @Override
    public boolean isDirectory(String filename) throws IOException {

        if (localRoot == null) {
            return filename.endsWith("/");
        }

        File f = new File(localRoot, filename);
        if (f.exists()) {
            return f.isDirectory();
        } else {
            if (filename.endsWith("/")) {
                return true;
            } else {
                File parentFile = f.getParentFile();
                String parent = getLocalName(parentFile);
                if (!parent.endsWith("/")) {
                    parent = parent + "/";
                }
                String[] list = listDirectory(parent);
                String lookFor;
                if (filename.startsWith("/")) {
                    lookFor = filename.substring(1) + "/";
                } else {
                    lookFor = filename + "/";
                }
                for (String list1 : list) {
                    if (list1.equals(lookFor)) {
                        return true;
                    }
                }
                return false;
            }
        }
    }

    /**
     * always hide these file types.
     * @return Arrays.asList( new String[] { ".css", ... } ).
     */
    private List<String> hideExtensions() {
        return Arrays.asList( new String[] { ".css", ".php", ".jnlp", ".part" } );
    }
    
    /**
     * list the directory, using the cached entry from listDirectoryFromMemory, or
     * by HtmlUtil.getDirectoryListing.  If there is a ro_cache, then add extra entries from here as well.
     * Note the following extentions are hidden: .css, .php, .jnlp, .part.
     * @param directory name within the filesystem
     * @return names within the directory
     * @throws IOException
     */
    @Override
    public String[] listDirectory(String directory) throws IOException {
        logger.log(Level.FINE, "** listDirectory({0}{1})", new Object[]{root, directory});

        DirectoryEntry[] cached= listDirectoryFromMemory( directory );
        if ( cached!=null ) {
            return FileSystem.getListing( cached );
        }

        if ( this.protocol!=null && this.protocol instanceof AppletHttpProtocol ) { // support applets.  This could check for local write access, but DefaultHttpProtocol shows a problem here too.
            URL[] list;
            try ( InputStream in = this.protocol.getInputStream( new WebFileObject(this,directory,new Date() ), new NullProgressMonitor() ) ) {
                list= HtmlUtil.getDirectoryListing( getURL(directory), in );
            } catch ( CancelledOperationException ex ) {
                throw new IllegalArgumentException(ex); //TODO: this should probably be IOException(ex).  See use 20 lines below as well.
            } 
            
            String[] result;
            result = new String[list.length];
            int n = directory.length();
            for (int i = 0; i < list.length; i++) {
                URL url = list[i];
                result[i] = getLocalName(url).substring(n);
            }
            return result;
        }

        directory = toCanonicalFolderName(directory);

        Map<String,DirectoryEntry> result;
        
        if ( isListingCached(directory) ) {
            logger.log(Level.FINER, "using cached listing for {0}", directory);

            File listing= listingFile(directory);
            
            URL[] list;
            try (FileInputStream fin = new FileInputStream(listing)) {
                list = HtmlUtil.getDirectoryListing(getURL(directory), fin );
            } catch (CancelledOperationException ex) {
                throw new IllegalArgumentException(ex); // shouldn't happen since it's local
            }
            
            result = new LinkedHashMap(list.length);
            int n = directory.length();
            for (URL url : list) {
                DirectoryEntry de1= new DirectoryEntry();
                de1.modified= Long.MAX_VALUE; // HTTP is somewhat expensive to get dates and sizes, so put in Long.MAX_VALUE to indicate need to load.
                de1.name= getLocalName(url).substring(n);
                de1.type= 'f'; //TODO: directories mis-marked?
                de1.size= Long.MAX_VALUE;
                result.put(de1.name,de1);
            }

            result= addRoCacheEntries( directory, result );

            cacheListing( directory, result.values().toArray( new DirectoryEntry[result.size()] ) );

            return FileSystem.getListing( result );
        }

        boolean successOrCancel= false;

        if ( this.isOffline() ) {
            File f= new File(localRoot, directory).getCanonicalFile();
            logger.log(Level.FINER, "this filesystem is offline, using local listing: {0}", f);

            result= addRoCacheEntries( directory, new LinkedHashMap() );
            
            if ( !f.exists() && result.isEmpty() ) throw new FileSystemOfflineException("unable to list "+f+" when offline");

            List<String> result1= new ArrayList();
            
            if ( f.exists() ) {
                File[] listing = f.listFiles();
                if ( listing==null ) {
                    throw new IllegalArgumentException("expected resource to be a directory: "+f);  
                } 

                for (File f1 : listing) {
                    if ( f1.getName().endsWith(".listing") ) continue;
                    if ( f1.isDirectory() ) {
                        result1.add( f1.getName() + '/' );
                    } else {
                        result1.add( f1.getName() );
                    }
                }
            }
            
            for ( DirectoryEntry f1: result.values() ) {
                if ( f1.type=='d' ) {
                    int n= f1.name.length();
                    if ( n>0 && f1.name.charAt(n-1)=='/' ) {
                        result1.add( f1.name );
                    } else {
                        result1.add( f1.name + '/' );
                    }
                } else {
                    result1.add( f1.name );
                }
            }
            return result1.toArray( new String[result1.size()] );
        }


        while ( !successOrCancel ) {
            logger.log(Level.FINER, "list {0}", directory);
            URL[] list;
            try {
                File listing= listingFile( directory );

                Map<String,String> metaz= downloadFile( directory, listing, getPartFile(listing), new NullProgressMonitor() );

                String code= metaz.get( WebProtocol.HTTP_RESPONSE_CODE );
                if ( code!=null && Integer.parseInt(code)==401 ) {
                    URL remoteURL = getURL( directory );
                    KeyChain.getDefault().clearUserPassword(remoteURL);
                    continue;
                }
                
                if ( !listing.setLastModified( System.currentTimeMillis() ) ) {
                    logger.log(Level.WARNING, "failed to setLastModified: {0}", listing);
                }
                
                try (FileInputStream fin = new FileInputStream(listing)) {
                    list = HtmlUtil.getDirectoryListing( getURL(directory), fin );
                }
                
                int n = FileSystemUtil.uriEncode(directory).length(); // note 20 lines above with getURL uriEncode is used in getURL
                
                //remove .css stuff
                ArrayList newlist= new ArrayList();
                List<String> hideExtensions= hideExtensions();
                for ( URL s: list ) {
                    boolean hide= false;
                    if ( !s.getFile().endsWith("/") ) {
                        for ( String e : hideExtensions ) {
                            if ( s.getFile().endsWith(e) ) hide= true;
                        }
                    }
                    try {
                        String ss= getLocalName(s).substring(n);
                        if ( ss.split("/").length>1 ) {
                            hide= true;
                        }
                    } catch ( IllegalArgumentException ex ) {
                        hide= true;
                    }
                    if ( !hide ) newlist.add(s);
                }
                list= (URL[]) newlist.toArray( new URL[newlist.size()] );

                result = new LinkedHashMap();
                for (URL url : list) {
                    DirectoryEntry de1= new DirectoryEntry();
                    de1.modified= Long.MAX_VALUE;
                    de1.name= getLocalName(url).substring(n);
                    de1.type= 'f';
                    de1.size= Long.MAX_VALUE;
                    result.put(de1.name,de1);
                }

                result= addRoCacheEntries( directory, result );
                cacheListing( directory, result.values().toArray( new DirectoryEntry[result.size()] ) );

                return FileSystem.getListing(result);
                
            } catch (CancelledOperationException ex) {
                throw new IOException( "user cancelled at credentials" ); // JAVA6
            } catch ( IOException ex ) {
                if ( isOffline() ) {
                    logger.info("** using local listing because remote is not available");
                    logger.info("or some other error occurred. **");
                    File localFile= new File( localRoot, directory );
                    return localFile.list();
                } else {
                    throw ex;
                }
            }

        }
        return( new String[] { "should not get here" } ); // we should not be able to reach this point
        
    }

//    public String[] listDirectoryOld(String directory) throws IOException {
//        directory = HttpFileSystem.toCanonicalFilename(directory);
//        if (!isDirectory(directory)) {
//            throw new IllegalArgumentException("is not a directory: " + directory);
//        }
//
//        if (!directory.endsWith("/")) {
//            directory = directory + "/";
//        }
//        synchronized (listings) {
//            if ( isListingCached(directory) ) { //TODO: there are no timestamps to invalidate listings!!!  How is it I haven't run across this before...https://sourceforge.net/tracker/index.php?func=detail&aid=3395693&group_id=199733&atid=970682
//                logger.log( Level.FINE, "use cached listing for {0}", directory );
//                String[] result= (String[]) listings.get(directory);
//                String[] resultc= new String[result.length];
//                System.arraycopy( result, 0, resultc, 0, result.length );
//                return resultc;
//
//            } else {
//                logger.log(Level.FINE, "list {0}", directory);
//                URL[] list;
//                try {
//                    list = HtmlUtil.getDirectoryListing(getURL(directory));
//                } catch (CancelledOperationException ex) {
//                    throw new IOException( "user cancelled at credentials" ); // JAVA6
//                } catch ( IOException ex ) {
//                    if ( isOffline() ) {
//                        System.err.println("** using local listing because remote is not available");
//                        System.err.println("or some other error occurred. **");
//                        File localFile= new File( localRoot, directory );
//                        return localFile.list();
//                    } else {
//                        throw ex;
//                    }
//                }
//                String[] result = new String[list.length];
//                int n = directory.length();
//                for (int i = 0; i < list.length; i++) {
//                    URL url = list[i];
//                    result[i] = getLocalName(url).substring(n);
//                }
//                listings.put(directory, result);
//                listingFreshness.put( directory, System.currentTimeMillis()+LISTING_TIMEOUT_MS );
//                return result;
//            }
//        }
//    }

    /**
     * return true if the regular expression for the file is actually a single
     * file with no wildcards.  This is tricky because for years a single period
     * (".") has been left in the name, so that "." matches ".", and really 
     * screen.png would match screen_png.  To contain this logic, this routine
     * is introduced.  Presently it just checks for "screen.png" so that a 
     * pngwalk of each screenshot (https://autoplot.org/jnlp/v$x/screen.png) can
     * be made, which is useful when looking for old versions.      
     * 
     * @param regex
     * @return 
     */
    public static boolean isRegexNoWild( String regex ) {
        return regex.equals("screen.png");
    }
    
    @Override
    public String[] listDirectory(String directory, String regex) throws IOException {
        
        //if ( SwingUtilities.isEventDispatchThread() ) {
            //logger.warning("listDirectory called on event thread!");
        //}
        
        logger.log(Level.FINE, "listDirectory({0},{1})", new Object[]{directory, regex});
        
        if ( regex.endsWith("/") ) regex= regex.substring(0,regex.length()-1);

        directory = toCanonicalFilename(directory);
        if (!isDirectory(directory)) {
            throw new IllegalArgumentException("is not a directory: " + directory);
        }

        if ( isRegexNoWild(regex) ) { // if it is not a regular expression // TODO: finally support ? in glob with """!regex.contains(".{1}")"""
            try {
                Map<String,Object> meta= getHeadMeta( directory+regex );
                if ( Boolean.TRUE.equals( meta.get(WebProtocol.META_EXIST) ) ) {
                    return new String[] { regex };
                }
            } catch (CancelledOperationException ex) {
                Logger.getLogger(HttpFileSystem.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
        
        String[] listing = listDirectory(directory);
        Pattern pattern = Pattern.compile(regex);
        ArrayList result = new ArrayList();
        for (String s : listing) {
            String c= s;
            if ( s.charAt(s.length()-1)=='/' ) c= s.substring(0,s.length()-1);
            if (pattern.matcher(c).matches()) {
                result.add(s);
            }
        }
        return (String[]) result.toArray(new String[result.size()]);

    }

    
    /**
     * HTTP listings are really done by querying the single file, so support this by issuing a head request
     * @param filename filename within the system
     * @param force if true, then guarantee a listing and throw an IOException if it cannot be done.
     * @return the DirectoryEntry showing size and date.
     * @throws IOException 
     */
    @Override
    public DirectoryEntry maybeUpdateDirectoryEntry(String filename, boolean force) throws IOException {
        Long fresh= listingEntryFreshness.get(filename);
        if ( fresh!=null ) {
            if ( new Date().getTime()-fresh < HttpFileSystem.HTTP_CHECK_TIMESTAMP_LIMIT_MS ) {
                return listingEntries.get(filename);
            } else {
                synchronized ( this ) {
                    listingEntryFreshness.remove(filename);
                    listingEntries.remove(filename);
                }
            }
        }
        try {
            Map<String,Object> meta= getHeadMeta(filename);
            DirectoryEntry de= new DirectoryEntry();
            String odate= (String)meta.get("Date");
            String osize= (String)meta.get("Content-Length");
            de.type= filename.endsWith("/") ? 'd' : 'f';
            if ( odate!=null && osize!=null ) {
                de.modified= new Date(odate).getTime();
                de.size= Long.parseLong(osize);
                synchronized ( this ) {
                    listingEntries.put(filename,de);
                    listingEntryFreshness.put(filename,new Date().getTime());
                }
                return de;
            } else {
                return super.maybeUpdateDirectoryEntry(filename, force);
            }
        } catch (CancelledOperationException ex) {
            return super.maybeUpdateDirectoryEntry(filename, force);
        }
    }
    
    
}
