/*
* FileStorageModel.java
*
* Created on March 31, 2004, 9:52 AM
*/
package org.das2.fsm;
import java.awt.EventQueue;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.text.ParseException;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.*;
import org.das2.DasApplication;
import org.das2.datum.CacheTag;
import org.das2.datum.Datum;
import org.das2.datum.DatumRange;
import org.das2.datum.DatumRangeUtil;
import org.das2.datum.TimeParser;
import org.das2.datum.TimeUtil.TimeStruct;
import org.das2.datum.Units;
import org.das2.util.LoggerManager;
import org.das2.util.filesystem.FileObject;
import org.das2.util.filesystem.FileSystem;
import org.das2.util.filesystem.FileSystemUtil;
import org.das2.util.filesystem.LocalFileSystem;
import org.das2.util.filesystem.SubFileSystem;
import org.das2.util.filesystem.WebFileSystem;
import org.das2.util.filesystem.ZipFileSystem;
import org.das2.util.monitor.NullProgressMonitor;
import org.das2.util.monitor.ProgressMonitor;
import org.das2.util.monitor.SubTaskMonitor;
/**
* Represents a method for storing data sets in a set of files by time. The
* client provides a regex for the files and how each group of the regex is
* interpreted as a time digit. The model can then be used to provide the set
* of files that cover a time range, etc.
*
* This new implementation uses a TimeParser object to more quickly process
* file names.
*
* @author Jeremy
*/
public class FileStorageModel {
private Pattern pattern;
private String regex;
private Pattern gzpattern;
FileStorageModel parent;
FileSystem root;
TimeParser timeParser;
String template;
static final Logger logger= LoggerManager.getLogger("das2.system.fsm");
HashMap fileNameMap=null;
private boolean allowGz= true; // if true, the getFile can use a .gz version to retrieve a file.
List<String> oldVersions= new ArrayList();
/**
* Versioning types
*/
static enum VersioningType {
none(null),
/**
* simple floating point numeric comparisons.
*/
numeric( new Comparator() { // 4.10 > 4.01
@Override
public int compare(Object o1, Object o2) {
Double d1= Double.parseDouble((String)o1);
Double d2= Double.parseDouble((String)o2);
return d1.compareTo(d2);
}
} ),
/**
* comparison by lexical sort v2013a>v2012b.
*/
alphanumeric(new Comparator() { // a001
@Override
public int compare(Object o1, Object o2) {
return ((String)o1).compareTo((String)o2);
}
} ),
/**
* comparison of numbers split by decimal points and dashes, so 1.20 > 1.3.
*/
numericSplit( new Comparator() { // 4.3.23 // 1.1.3-01 for RBSP (rbspice lev-2 isrhelt)
@Override
public int compare(Object o1, Object o2) {
String[] ss1= o1.toString().split("[\\.-]",-2);
String[] ss2= o2.toString().split("[\\.-]",-2);
int n= Math.min( ss1.length, ss2.length );
for ( int i=0; i<n; i++ ) {
int d1= Integer.parseInt(ss1[i]);
int d2= Integer.parseInt(ss2[i]);
if ( d1!=d2 ) {
return d1 < d2 ? -1 : 1;
}
}
return ss1.length - ss2.length; // the longer version wins (3.2.1 > 3.2)
}
});
Comparator<String> comp;
VersioningType( Comparator<String> comp ) {
this.comp= comp;
}
};
VersioningType versioningType;
String versionGe= null; // the version must be greater than or equal to this if non-null.
String versionLt= null; // the version must be less than this if non-null.
/**
* return the filesystem used to implement this.
* @return filesystem
*/
public FileSystem getFileSystem() {
return this.root;
}
/**
* return a child filesystem, with special code for LocalFileSystems to support
* Windows. TODO: look into .zip child.
* @param root
* @param child
* @param monitor
* @return the FileSystem
* @see https://sourceforge.net/p/autoplot/bugs/2132
* @throws org.das2.util.filesystem.FileSystem.FileSystemOfflineException
* @throws UnknownHostException
* @throws FileNotFoundException
*/
public static FileSystem getChildFileSystem( FileSystem root, String child, ProgressMonitor monitor ) throws FileSystem.FileSystemOfflineException, UnknownHostException, FileNotFoundException {
FileSystem result;
if ( root.getRootURI().getScheme().equals("file") ) {
if ( root instanceof LocalFileSystem ) {
File localRoot= ((LocalFileSystem)root).getLocalRoot();
result= FileSystem.create( new File( localRoot, child ).toURI(), monitor.getSubtaskMonitor("create") );
} else if ( root instanceof SubFileSystem ) {
try {
result= root.createFileSystem(child);
} catch (URISyntaxException ex) {
throw new IllegalArgumentException(ex);
}
} else if ( root instanceof ZipFileSystem ) {
try {
result= root.createFileSystem(child); // ((ZipFileSystem)root).createFileSystem(child);
} catch (URISyntaxException ex) {
throw new IllegalArgumentException(ex);
}
} else {
throw new IllegalArgumentException("not supported, should not happen");
}
} else {
result= FileSystem.create( root.getRootURI().resolve( child ), monitor.getSubtaskMonitor("create") ); // 3523492: allow the FS type to change; eg to zip.
}
return result;
//fileSystems[i]= root.createFileSystem( names[i] ); //TODO: look into this.
// 3523492: allow the FS type to change; eg to zip.
}
/**
* return a random file from the FSM, which can be used to represent a typical file. For
* example, we need to look at metadata to see what is available.
* @param monitor progress monitor in case a file must be downloaded.
* @return a reference to the file within the FileSystem, or null if one is not found.
* @throws IOException
*/
public String getRepresentativeFile( ProgressMonitor monitor ) throws IOException {
return getRepresentativeFile( this, monitor, null, null, 0 );
}
/**
* return a random file from the FSM, which can be used to represent a typical file. For
* example, we need to look at metadata to see what is available.
* @param monitor progress monitor in case a file must be downloaded.
* @param childRegex the parent must contain a file/folder matching childRegex
* @return a reference to the file within the FileSystem, or null if one is not found.
* @throws IOException
*/
public String getRepresentativeFile( ProgressMonitor monitor, String childRegex ) throws IOException {
return getRepresentativeFile( this, monitor, childRegex, null, 0 );
}
/**
* Return a random file from the FSM, which can be used to represent a typical file. For
* example, we need to look at metadata to see what is available. This is introduced
* to support discovery, where we just need one file to
* get started. Before, there was code that would list all files, then use
* just the first one. This may return a skeleton file, but getFileFor() must
* return a result.
* This implementation does the same as getNames(), but stops after finding a file.
* @param monitor progress monitor in case a file must be downloaded.
* @param childRegex the parent must contain a file/folder matching childRegex, or null if there is no child.
* @param range hint at the range where we are looking, or null if there is not time range constraint.
* @return null if no file is found
* @throws java.io.IOException if the file cannot be downloaded.
*/
public String getRepresentativeFile( ProgressMonitor monitor, String childRegex, DatumRange range ) throws IOException {
return getRepresentativeFile( this, monitor,childRegex,range,0 );
}
/**
* Return a random file from the FSM, which can be used to represent a typical file. For
* example, we need to look at metadata of a given file to see what is available inside.
* This is introduced to support discovery, where we just need one file to
* get started. Before, there was code that would list all files, then use
* just the first one. This may return a skeleton file, but getFileFor() must
* return a result.
* This implementation does the same as getNames(), but stops after finding a file.
* @param ths
* @param monitor progress monitor in case a file must be downloaded.
* @param childRegex the parent must contain a file/folder matching childRegex, or null if there is no child.
* @param range hint at the range where we are looking.
* @param depth the recursion depth, useful for debugging.
* @return null if no file is found
* @throws java.io.IOException if the file cannot be downloaded.
*/
public static String getRepresentativeFile( FileStorageModel ths, ProgressMonitor monitor, String childRegex, DatumRange range, int depth ) throws IOException {
logger.log(Level.FINE, "get representative from {0} {1} range: {2}", new Object[]{ths.getFileSystem(), childRegex, range});
//if ( depth==1 ) { //TODO: recusion makes this really hard to debug, and this should be rewritten to remove recursion.
// System.err.println("here at depth 1: "+ths.toString() );
//}
if ( monitor==null ) monitor= new NullProgressMonitor();
String listRegex;
FileSystem[] fileSystems;
String[] names;
String parentRegex=null;
DatumRange range1; // the range from the parent we are looking for. This is to limit searches...
if ( EventQueue.isDispatchThread() ) {
logger.info("FileSystem use on the GUI event thread will often cause problems.");
//new Exception("FileSystem uses event thread stack trace").printStackTrace();
}
String result= null;
try {
if ( ths.parent!=null ) {
parentRegex= getParentRegex(ths.regex);
String one= getRepresentativeFile( ths.parent, monitor.getSubtaskMonitor("get representative file"),ths.regex.substring(parentRegex.length()+1), range, depth+1 );
if ( one==null ) return null;
names= new String[] { one }; //parent.getNamesFor(null);
fileSystems= new FileSystem[names.length];
for ( int i=0; i<names.length; i++ ) {
try {
fileSystems[i]= getChildFileSystem( ths.root, names[i], monitor.getSubtaskMonitor("create") );
} catch ( FileSystem.FileSystemOfflineException | UnknownHostException | FileNotFoundException e ) {
throw new RuntimeException(e);
}
}
listRegex= ths.regex.substring( parentRegex.length()+1 );
} else {
fileSystems= new FileSystem[] { ths.root };
names= new String[] {""};
listRegex= ths.regex;
}
while ( result==null ) {
for ( int i=fileSystems.length-1; result==null && i>=0; i-- ) {
String[] files1= fileSystems[i].listDirectory( "/", listRegex, monitor.getSubtaskMonitor("create") );
//to avoid the case where the latest folder is empty, check n-1 down to 0, then n.
//int n= files1.length;
//String[] resort= new String[n];
//for ( int j=0; j<n-1; j++ ) {
// resort[j]= files1[n-2-j];
//}
//resort[n-1]= files1[n-1];
//files1= resort;
//for ( int j=0; j<files1.length && result==null; j++ ) {
int j= files1.length-1;
while ( j>=0 && result==null ) {
String ff= names[i].equals("") ? files1[ j ] : names[i]+"/"+files1[ j ];
if ( ff.endsWith("/") ) ff=ff.substring(0,ff.length()-1);
try {
HashMap<String,String> extra= new HashMap();
DatumRange tr= getDatumRangeFor( ths, ff, extra );
boolean versionOk= true;
if ( ths.versionGe!=null && ths.versioningType.comp.compare( extra.get("v"), ths.versionGe )<0 ) versionOk=false;
if ( ths.versionLt!=null && ths.versioningType.comp.compare( extra.get("v"), ths.versionLt )>=0 ) versionOk=false;
if ( versionOk && ths.timeParser.getValidRange().contains( tr ) && ( range==null || range.intersects(tr) ) ) {
if ( childRegex!=null ) {
String[] kids= fileSystems[i].listDirectory( files1[ j ],childRegex, monitor.getSubtaskMonitor("list directory") );
if ( kids.length>0 ) {
result= ff;
}
} else {
result= ff;
}
}
} catch ( IllegalArgumentException ex ) {
logger.log( Level.FINER, null, ex );
}
if ( result==null ) j--;
}
}
if ( ths.allowGz ) {
if ( result==null) {
for ( int i=fileSystems.length-1; result==null && i>=0; i-- ) {
String[] files1= fileSystems[i].listDirectory( "/", listRegex + ".gz" );
if ( files1.length>0 ) {
int last= files1.length-1;
String ff= names[i].equals("") ? files1[ last ] : names[i]+"/"+files1[ last ];
if ( ff.endsWith("/") ) ff=ff.substring(0,ff.length()-1);
result= ff.substring( 0,ff.length()-3 );
}
}
}
}
if ( result==null ) {
if ( ths.parent==null ) {
return null;
} else { // fall back to old code that would list everything.
logger.fine("fall back to old code that would list everything");
range1= ths.parent.getRangeFor(names[0]);
range1= range1.previous();
if ( range!=null && !range.intersects(range1) ) {
return null;
}
String one= getRepresentativeFile(
ths.parent,
monitor.getSubtaskMonitor("getRepresentativeFile"),
ths.regex.substring(parentRegex.length()+1),
range1,
depth+1 );
if ( one==null ) return null;
names= new String[] { one }; //parent.getNamesFor(null);
fileSystems= new FileSystem[names.length];
for ( int i=0; i<names.length; i++ ) {
try {
fileSystems[i]= getChildFileSystem( ths.root, names[i], monitor.getSubtaskMonitor("create") );
} catch ( Exception e ) {
throw new RuntimeException(e);
}
}
}
}
}
} finally {
monitor.finished();
}
return result;
}
/**
* set the datum range giving context to the files. For example,
* filenames are just $H$M$S.dat, and the context is "Jan 17th, 2015"
* @param trdr the context
*/
public void setContext( DatumRange trdr ) {
timeParser.setContext(trdr);
}
/**
* extract time range for file or directory from its name.
* The least significant time digit is considered to be the implicitTimeWidth,
* and if the width is not stated explicitly, it will be used. When
* a set timeDigits are encountered twice, then the second occurrence
* is considered be the end time.
*
* .../FULL1/T8709_12/T871118.DAT
*'.../FULL1/T'YYMM_MM/TYYMMDD'.DAT' TODO: verify this.
* .../FULL1/T$y$m_$m/T$y$m$d.dat
*
* @param filename the filename, within the filesystem.
* @param extra an empty map where extra fields (such as version) are put.
* @throws IllegalArgumentException when a filename does not match the model specification.
*/
private synchronized DatumRange getDatumRangeFor( String filename, Map<String,String> extra ) {
try {
extra.clear();
if ( pattern.matcher(filename).matches() ) {
timeParser.parse( filename, extra );
return timeParser.getTimeRange();
} else {
if ( gzpattern!=null && gzpattern.matcher(filename).matches() ) {
timeParser.parse( filename.substring(0,filename.length()-3), extra );
return timeParser.getTimeRange();
} else {
throw new IllegalArgumentException( "file name \""+filename+"\" doesn't match model specification ("+template+")");
}
}
} catch ( ParseException | NumberFormatException e ) {
IllegalArgumentException e2=new IllegalArgumentException( "file name \""+filename+"\" doesn't match model specification ("+template+"), parse error in field",e);
throw e2;
}
}
private static DatumRange getDatumRangeFor( FileStorageModel ths, String filename, Map<String,String> extra ) {
try {
extra.clear();
if ( ths.pattern.matcher(filename).matches() ) {
ths.timeParser.parse( filename, extra );
return ths.timeParser.getTimeRange();
} else {
if ( ths.gzpattern!=null && ths.gzpattern.matcher(filename).matches() ) {
ths.timeParser.parse( filename.substring(0,filename.length()-3), extra );
return ths.timeParser.getTimeRange();
} else {
throw new IllegalArgumentException( "file name \""+filename+"\" doesn't match model specification ("+ths.template+")");
}
}
} catch ( ParseException | NumberFormatException e ) {
IllegalArgumentException e2=new IllegalArgumentException( "file name \""+filename+"\" doesn't match model specification ("+ths.template+"), parse error in field",e);
throw e2;
}
}
/**
* return a filename that would intersect the range. Note this file
* may not actually exist. This may be used to quantize the range.
* The template may not contain versions.
* @param start
* @param end
* @return
*/
public String getFilenameFor( Datum start, Datum end ) {
return timeParser.format( start, end );
}
/**
* generate the names of the files that would cover this range. This was
* taken from Autoplot's org.virbo.jythonsupport.Util.
* TODO: versioning, etc.
* @param range the time range to cover.
* @return the string names, each in the context of the filesystem.
*/
public String[] generateNamesFor( DatumRange range ) {
String sstart;
TimeParser tp= timeParser;
try {
sstart= tp.format( range.min(), null );
} catch ( Exception ex ) { // orbit files have limited range
DatumRange dr= tp.getValidRange();
DatumRange dd= DatumRangeUtil.sloppyIntersection(range, dr);
if ( dd.width().value()==0 ) {
return new String[0]; // no intersection
}
sstart= tp.format( dd.min(), null );
}
try {
tp.parse(sstart);
} catch ( ParseException ex ) {
throw new RuntimeException(ex);
}
DatumRange curr= tp.getTimeRange();
int countLimit= 1000000;
int approxCount= (int)( 1.01 * range.width().divide(curr.width()).value() ); // extra 1% for good measure.
if ( approxCount>countLimit*1.03 ) {
throw new IllegalArgumentException("too many intervals would be created, this is limited to about 1000000 intervals.");
}
List<String> result= new ArrayList<>( approxCount );
if ( !range.intersects(curr) ) { // Sebastian has a strange case that failed, see
curr= curr.next();
}
while ( range.intersects(curr) ) {
String scurr= tp.format( curr.min(), curr.max() );
result.add( scurr );
DatumRange oldCurr= curr;
curr= curr.next();
if ( oldCurr.equals(curr) ) { // orbits return next() that is this at the ends.
break;
}
}
return result.toArray( new String[result.size()] );
}
/**
* return the timerange that contains the given timerange and
* exactly contains a set of granules.
*
* This needs to be synchronized because the timeParser.
* @param timeRange arbitrary time range
* @return list of file timeranges covering file input timeRange.
*/
public synchronized DatumRange quantize(DatumRange timeRange) {
try {
String tf1= timeParser.format( timeRange.min(), timeRange.min() );
String tf2= timeParser.format( timeRange.max(), timeRange.max() );
DatumRange dr1,dr2;
try {
dr1= timeParser.parse(tf1).getTimeRange();
dr2= timeParser.parse(tf2).getTimeRange();
// if ( true ) {
// TimeParser tp1= timeParser.parse(tf1);
// logger.log( Level.WARNING, "tp1 start {0}", tp1.getTime(Units.us2000) );
// logger.log( Level.WARNING, "tp1 end {0}", tp1.getEndTime(Units.us2000) );
// logger.log( Level.WARNING, "tp1 tr {0}", tp1.getTimeRange() );
// TimeParser tp2= timeParser.parse(tf2);
// logger.log( Level.WARNING, "tp2 start {0}", tp2.getTime(Units.us2000) );
// logger.log( Level.WARNING, "tp2 end {0}", tp2.getEndTime(Units.us2000) );
// logger.log( Level.WARNING, "tp2 tr {0}", tp2.getTimeRange() );
// }
} catch ( IllegalArgumentException ex ) {
logger.log(Level.WARNING, "Strange bug shown in test033: {2}\n>>{0}<<\n>>{1}<<", new Object[]{tf1, tf2,this.timeParser});
TimeParser tp1= timeParser.parse(tf1);
logger.log( Level.WARNING, "tp1 start {0}", tp1.getTime(Units.us2000) );
logger.log( Level.WARNING, "tp1 end {0}", tp1.getEndTime(Units.us2000) );
logger.log( Level.WARNING, "tp1 tr {0}", tp1.getTimeRange() );
TimeParser tp2= timeParser.parse(tf2);
logger.log( Level.WARNING, "tp2 start {0}", tp2.getTime(Units.us2000) );
logger.log( Level.WARNING, "tp2 end {0}", tp2.getEndTime(Units.us2000) );
logger.log( Level.WARNING, "tp2 tr {0}", tp2.getTimeRange() );
throw ex;
}
if ( dr2.min().equals(timeRange.max() ) ) {
return DatumRangeUtil.union( dr1, dr2.min() );
} else {
return DatumRangeUtil.union( dr1, dr2 );
}
} catch (ParseException ex) {
throw new RuntimeException("this shouldn't happen");
}
}
/**
* return the names in the range, or all names if the range is null.
* @param targetRange range limit, or null.
* @return array of names within the system.
* @throws java.io.IOException
*/
public String[] getNamesFor( final DatumRange targetRange ) throws IOException {
return getNamesFor( targetRange, false, new NullProgressMonitor() );
}
/**
* return the names in the range, or all available names if the range is null.
* This will list directories.
* @param targetRange range limit, or null.
* @param monitor
* @return array of names within the system.
* @throws java.io.IOException
*/
public String[] getNamesFor( final DatumRange targetRange, ProgressMonitor monitor ) throws IOException {
return getNamesFor( targetRange, false, monitor );
}
/**
* return the names in the range, minding version numbers, or all available
* names if the range is null. This will list directories.
* @param targetRange range limit, or null.
* @param monitor
* @return array of names within the system.
* @throws java.io.IOException
*/
public String[] getBestNamesFor( final DatumRange targetRange, ProgressMonitor monitor ) throws IOException {
return getNamesFor( targetRange, true, monitor );
}
/**
* return the names in the range, minding version numbers, or all available
* names if the range is null. This will list directories.
* @param targetRange range limit, or null.
* @return array of names within the system.
* @throws java.io.IOException
*/
public String[] getBestNamesFor( final DatumRange targetRange ) throws IOException {
return getNamesFor( targetRange, true, new NullProgressMonitor() );
}
/**
* return the names in the range, maybe with versioning. This will
* list directories.
* @param targetRange range limit, or null if no constraint used here.
* @param versioning true means check versions.
* @param monitor progress monitor (or null)
* @return array of names within the system.
* @throws IOException
* @see generateNamesFor
*/
private String[] getNamesFor( final DatumRange targetRange, boolean versioning, ProgressMonitor monitor ) throws IOException {
logger.log( Level.FINE, "getNamesFor {0}", this.root);
if ( monitor==null ) monitor= new NullProgressMonitor();
String listRegex;
FileSystem[] fileSystems;
String[] names;
oldVersions.clear();
if ( parent!=null ) {
names= parent.getNamesFor(targetRange,versioning,new NullProgressMonitor()); // note recursive call
logger.log(Level.FINE, "parent {0} yields: {1}", new Object[]{parent.toString(), names.length});
fileSystems= new FileSystem[names.length];
for ( int i=0; i<names.length; i++ ) {
try {
fileSystems[i]= getChildFileSystem( root, names[i], monitor );
} catch ( Exception e ) {
throw new RuntimeException(e);
}
}
String parentRegex= getParentRegex(regex); //vap:ftp://virbo.org/POES/n15/$Y/poes_n15_$Y$m$d.cdf.zip/poes_n15_$Y$m$d.cdf?minute&timerange=1998-07-01
listRegex= regex.substring( parentRegex.length()+1 );
} else {
fileSystems= new FileSystem[] { root };
names= new String[] {""};
listRegex= regex;
}
List<String> list= new ArrayList();
List<String> versionList= new ArrayList();
List<DatumRange> rangeList= new ArrayList();
monitor.setTaskSize( fileSystems.length*10 );
monitor.started();
Map<String,String> extra= new HashMap();
for ( int i=0; i<fileSystems.length; i++ ) {
if ( monitor.isCancelled() ) {
throw new InterruptedIOException("cancel pressed");
}
monitor.setTaskProgress(i*10);
// This is a nice trick for debugging when there is recursion and NullProgressMonitor is not passed in.
// if ( !( monitor instanceof NullProgressMonitor ) ) {
// System.err.println("here");
// }
String theListRegex= listRegex;
if ( this.allowGz ) {
theListRegex= theListRegex+"(.gz)?";
}
String[] files1= fileSystems[i].listDirectory( "/", theListRegex );
logger.log( Level.FINER, "listDirectory({0})->{1}", new Object[]{theListRegex, files1.length});
Arrays.sort(files1);
for ( int j=0; j<files1.length; j++ ) {
String ff= names[i].equals("") ? files1[j] : names[i]+"/"+files1[j];
if ( ff.endsWith("/") ) ff=ff.substring(0,ff.length()-1);
try {
DatumRange dr= getDatumRangeFor( ff, extra );
if ( targetRange==null || dr.intersects(targetRange) ) {
if ( ff.endsWith(".gz") && this.allowGz ) { //TODO: make sure allowgz is off when we really want gz files.
ff= ff.substring(0,ff.length()-3);
}
list.add(ff);
rangeList.add(dr);
if ( versioningType!=VersioningType.none ) {
if ( extra.get("v")==null ) throw new RuntimeException("expected version");
versionList.add( extra.get("v") );
}
logger.log( Level.FINER, " add {0}", ff);
}
} catch ( IllegalArgumentException e ) {
logger.log( Level.FINER, " skip {0} because it does not parse properly", ff ); // this just means file doesn't match template.
}
monitor.setTaskProgress( i*10 + j * 10 / files1.length );
}
if ( fileSystems[i] instanceof WebFileSystem ) {
WebFileSystem wfs= (WebFileSystem)fileSystems[i];
File f= wfs.getLocalRoot();
if ( f!=null ) { // e.g. Applet support, where applets have no local root.
FileSystem lfs= FileSystem.create( f.toURI() );
String[] files2= lfs.listDirectory( "/", theListRegex );
List<String> deleteRemote= new ArrayList();
List<String> remoteFiles= Arrays.asList( files1 );
for ( String s: files2 ) {
if ( remoteFiles.indexOf(s)==-1 ) {
deleteRemote.add(s);
}
}
logger.log(Level.FINE, "local files that do not exist on remote: {0}", deleteRemote);
oldVersions.addAll(deleteRemote);
}
}
}
if ( versioning && versioningType!=VersioningType.none) {
Comparator<String> comp= versioningType.comp;
Map<String,String> bestVersions= new HashMap();
Map<String,String> bestFiles= new HashMap();
for ( int j=0; j<list.size(); j++ ) {
String ff= list.get(j);
String key= rangeList.get(j).toString();
String thss= versionList.get(j);
String best= bestVersions.get(key);
if ( best==null ) {
try {
logger.log(Level.FINER, "check format exception: {0}", comp.compare( thss, thss )); // check for format exception
if ( ( versionGe==null || comp.compare( thss, versionGe )>=0 )
&& ( versionLt==null || comp.compare( thss, versionLt )<0 ) ){
bestVersions.put( key, thss );
bestFiles.put( key,ff );
}
} catch ( Exception ex ) {
logger.log( Level.WARNING, ex.getMessage(), ex );
// doesn't match if comparator (e.g. version isn't a decimal number)
}
} else {
try {
if ( ( ( versionGe==null || comp.compare( thss, versionGe )>=0 ) )
&& ( versionLt==null || comp.compare( thss, versionLt )<0 )
&& comp.compare( thss, best ) > 0 ) {
bestVersions.put( key,thss );
String rm= bestFiles.put( key,ff );
if ( !oldVersions.contains(rm) ) {
oldVersions.add(rm);
}
}
} catch ( Exception ex ) {
logger.log( Level.WARNING, ex.getMessage(), ex );
// doesn't match
}
}
}
list= Arrays.asList( bestFiles.values().toArray( new String[ bestFiles.size()] ) );
}
Collections.sort( list, new Comparator() {
@Override
public int compare( Object o1, Object o2 ) {
DatumRange dr1= getRangeFor( (String)o1 );
DatumRange dr2= getRangeFor( (String)o2 );
return dr1.compareTo( dr2 );
}
} );
logger.log( Level.FINE, "getNamesFor {0} -> {1}", new Object[] { this.root, list.size() } );
monitor.finished();
return (String[])list.toArray(new String[list.size()]);
}
public static CacheTag getCacheTagFor( FileStorageModel fsm, DatumRange range, String[] names ) {
Datum min= range.min();
Datum max= range.max();
for (String name : names) {
DatumRange r = fsm.getRangeFor(name);
min= min.gt(range.min()) ? r.min() : min;
max= max.lt(range.max()) ? r.max() : max;
}
return new CacheTag( min, max, null );
}
public static CacheTag getCacheTagFor( FileStorageModel fsm, DatumRange range, File[] files ) {
String[] names= new String[files.length];
for ( int i=0; i<files.length; i++ ) {
names[i]= fsm.getNameFor(files[i]);
}
return getCacheTagFor( fsm, range, names );
}
public File[] getFilesFor( final DatumRange targetRange ) throws IOException {
return getFilesFor( targetRange, new NullProgressMonitor() );
}
/**
* return the best files found for the range, without progress feedback.
* @param targetRange
* @return
* @throws IOException
*/
public File[] getBestFilesFor( final DatumRange targetRange ) throws IOException {
return getBestFilesFor( targetRange, new NullProgressMonitor() );
}
/**
* remove files that have been identified as old versions.
*/
public void cacheCleanup() {
if ( getFileSystem() instanceof LocalFileSystem ) {
logger.fine("local filesystems do not cache");
} else {
for ( String s: oldVersions ) {
if ( getFileSystem().getFileObject(s).removeLocalFile()==false ) {
logger.log(Level.FINER, "removeLocalFile returned false: {0}", s);
}
}
}
}
/**
* return the time range represented by this name.
* @param name like 2013-10-31
* @return the timerange representing the day 2013-10-31
* @throws IllegalArgumentException if the name is not part of the FileStorageModel.
*/
public DatumRange getRangeFor( String name ) {
return getDatumRangeFor( name, new HashMap() );
}
/**
* returns true if the parser has the field.
* @param field e.g. "x"
* @return true if the parser has the field.
*/
public boolean hasField( String field ) {
return timeParser.hasField( field );
}
/**
* return the field value for the given name. For example, if the spec
* is $Y/$m/$d/$Y$m$d_v$(v,sep).dat and the name matched is 2014/04/04/20140404_v2.3.dat
* then calling this for the field "v" would result in "2.3" This should not
* be used to retrieve fields that are components of the time range, such as
* $Y or $m.
* @param field field, for example "v"
* @param name name, for example 2014/04/04/20140404_v2.3.dat
* @return the field value, for example, "2.3" when the spec is $Y/$m/$d/$Y$m$d_v$v.dat
*/
public String getField( String field, String name ) {
HashMap<String,String> hh= new HashMap();
getDatumRangeFor( name, hh );
if ( hh.containsKey(field) ) {
return hh.get(field);
} else {
throw new IllegalArgumentException("field is not in template: "+field);
}
}
/**
* return true if the file came (or could come) from this FileStorageModel.
* @param file the file
* @return true if the file came (or could come) from this FileStorageModel.
*/
public boolean containsFile( File file ) {
if ( !fileNameMap.containsKey(file) ) {
return false;
} else {
String name= getNameFor( file );
return containsName(name);
}
}
/**
* return true if the name came (or could come) from this FileStorageModel.
* @param name the name within the filesystem.
* @return true if the name came (or could come) from this FileStorageModel.
*/
public boolean containsName( String name ) {
Matcher m= pattern.matcher( name );
return m.matches();
}
/**
* Provides a way to recover the model name of a file. The returned File from getFilesFor can be anywhere,
* so it would be good to provide a way to get it back into a FSM name. For example, a filesystem
* might download the remote file to a cache directory, which is the File that is provided to the
* client, sometimes the client will need to recover the name of the corresponding FileObject, so
* this maps the File back to the name.
*
* @param file the file
* @return the canonical name of the file.
*/
public String getNameFor( File file ) {
String result= (String)fileNameMap.get(file);
if ( result==null ) {
throw new IllegalArgumentException( "File didn't come from this FileStorageModel" );
} else {
return result;
}
}
/**
* return the root of the filesystem as a string.
* @return the root of the filesystem as a string.
*/
public String getRoot() {
return root.getRootURI().toString();
}
/**
* check to see if "NAME.gz" exists
* @param name name of uncompressed file
* @param mon progress monitor (or null)
* @return null or the uncompressed file.
* @throws IOException
*/
private File maybeGetGzFile( String name, ProgressMonitor mon) throws IOException {
File f0 = null;
if ( mon==null ) mon= new NullProgressMonitor();
FileObject oz = root.getFileObject(name + ".gz");
if (oz.exists()) {
File fz = oz.getFile(mon);
String sfz = fz.getPath().substring(0, fz.getPath().length() - 3);
f0 = new File(sfz);
FileSystemUtil.gunzip(fz, f0);
if ( !f0.setLastModified(fz.lastModified()) ) {
throw new IllegalArgumentException("failed to set last modified");
}
}
return f0;
}
/**
* download the file for the given name within the filesystem.
* @param name the name within the filesystem.
* @return null or a local file which can be opened.
* @throws IOException
*/
public File getFileFor( String name ) throws IOException {
File[] ff= getFilesFor( new String[] { name }, new NullProgressMonitor() );
if ( ff.length>0 ) {
return ff[0];
} else {
return null;
}
}
/**
* download the file for the given name within the filesystem.
* @param name the name within the filesystem.
* @param monitor monitor for the download.
* @return null or a local file which can be opened.
* @throws IOException
*/
public File getFileFor( String name, ProgressMonitor monitor ) throws IOException {
File[] ff= getFilesFor( new String[] { name }, monitor );
if ( ff.length>0 ) {
return ff[0];
} else {
return null;
}
}
/**
* download the files for each of the given names within the filesystem.
* @param names array of names within the filesystem
* @param monitor monitor for the downloads (or null).
* @return local files that can be opened.
* @throws java.io.IOException during the transfer
*/
public File[] getFilesFor( String [] names, ProgressMonitor monitor ) throws IOException {
if ( monitor==null ) monitor= new NullProgressMonitor();
File[] files= new File[names.length];
if ( fileNameMap==null ) fileNameMap= new HashMap();
int numwarn= 0;
if ( names.length>0 ) monitor.setTaskSize( names.length * 10 );
monitor.started();
for ( int i=0; i<names.length; i++ ) {
if ( monitor.isCancelled() ) {
throw new InterruptedIOException("cancel pressed");
}
try {
FileObject o= root.getFileObject( names[i] );
if ( o.exists() ) {
files[i]= o.getFile( SubTaskMonitor.create( monitor, i*10, (i+1)*10 ));
} else if ( allowGz ) {
File f0 = maybeGetGzFile( names[i], SubTaskMonitor.create(monitor, i * 10, (i + 1) * 10) );
files[i]= f0;
}
if ( files[i]==null && numwarn<3 ) {
logger.log(Level.WARNING, "listing returns result that cannot be resolved file file (e.g. bad link): {0}", names[i]);
numwarn++;
}
fileNameMap.put( files[i], names[i] );
} catch ( IOException e ) {
throw new RuntimeException(e);
}
}
monitor.finished();
// remove nulls that come from bad references.
ArrayList<File> result= new ArrayList(files.length);
int i=0;
for (File file : files) {
if (file != null) {
result.add(file);
i++;
}
}
return result.toArray( new File[i] );
}
/**
* Download the files within the range.
* This might catch a bad link where getNamesFor does not.
* @param targetRange range limit, or null if no constraint used here.
* @param monitor the monitor
* @return a list of files that can be opened.
* @throws java.io.IOException
*/
public File[] getFilesFor( final DatumRange targetRange, ProgressMonitor monitor ) throws IOException {
String[] names= getNamesFor( targetRange );
File[] ff= getFilesFor( names, monitor );
return ff;
}
/**
* Get the files for the range, using versioning info ($v,etc).
* @param targetRange range limit, or null if no constraint used here.
* @param monitor the monitor
* @return a list of files that can be opened.
* @throws IOException
*/
public File[] getBestFilesFor( final DatumRange targetRange, ProgressMonitor monitor ) throws IOException {
if ( monitor==null ) monitor= new NullProgressMonitor();
String[] names= getNamesFor( targetRange, true, monitor.getSubtaskMonitor("get names") );
File[] files= new File[names.length];
if ( fileNameMap==null ) fileNameMap= new HashMap();
if ( names.length>0 ) monitor.setTaskSize( names.length * 10 );
monitor.started();
for ( int i=0; i<names.length; i++ ) {
if ( monitor.isCancelled() ) {
throw new InterruptedIOException("cancel pressed");
}
try {
FileObject o= root.getFileObject( names[i] );
if ( o.exists() ) {
files[i]= o.getFile( SubTaskMonitor.create( monitor, i*10, (i+1)*10 ));
} else if ( allowGz ) {
File f0 = maybeGetGzFile( names[i], SubTaskMonitor.create(monitor, i * 10, (i + 1) * 10) );
files[i]= f0;
}
fileNameMap.put( files[i], names[i] );
} catch ( IOException e ) {
throw new RuntimeException(e);
}
}
monitor.finished();
return files;
}
private static int countGroups( String regex ) {
int result=0;
Pattern p= Pattern.compile( regex );
Matcher m= p.matcher("");
return m.groupCount();
}
/**
* Split off the end of the regex to get a regex for use in the parent system. Returns null
* if there is no parent portion of the regex.
*/
private static String getParentRegex( String regex ) {
String[] s= regex.split( "/" );
StringBuilder dirRegex;
if ( s.length>1 ) {
dirRegex= new StringBuilder( s[0] );
for ( int i=1; i<s.length-1; i++ ) {
dirRegex.append( "/" ) .append( s[i] );
}
} else {
dirRegex= null;
}
return dirRegex==null ? null : dirRegex.toString();
}
/**
* returns the parent or null if none exists. None will exist when there
* are no more wildcards. (/home/foo/$Y/$m-$d.dat has parent /home/foo/$Y
* which has parent null.)
*
* @return parent or null.
*/
public FileStorageModel getParent() {
return this.parent;
}
/**
* Autoplot introduced the dollar sign instead of the percent, because $ is
* more URI-friendly. Switch to this if it looks appropriate.
*
* NOTE: this does not correct commas in the qualifiers section. TODO: fix this!
*
* @param template, e.g "/%Y/%m%d.dat"
* @return "$Y/$m$d.dat"
*/
protected static String makeCanonical( String template ) {
String result;
if ( template.contains("$Y") || template.contains("$y" ) ) {
result= template.replaceAll("\\$", "%");
} else {
result= template;
}
result= result.replaceAll("//+","/");
int i=result.indexOf("/");
if ( i>-1 && result.indexOf("%")>i ) {
System.err.println("each folder of template must have fields marked by $ or %: "+ result.substring(0,i) );
}
if ( result.startsWith("/") ) {
result= result.substring(1);
}
return result;
}
/**
* hide the contents of parameters as in<pre>
* product_%(o,id=ftp://stevens.lanl.gov/pub/projects/rbsp/autoplot/orbits/rbspa_pp).png -->
* product_%(______________________________________________________________________).png
* </pre>
* @param template
* @return
*/
private static String hideParams( String template ) {
StringBuilder result= new StringBuilder();
boolean withinArg= false;
for ( int i=0; i<template.length(); i++ ) {
if ( withinArg ) {
if ( template.charAt(i)==')' ) withinArg= false;
if ( withinArg ) result.append("_"); else result.append(template.charAt(i) );
} else {
if ( template.charAt(i)=='(' ) withinArg= true;
result.append(template.charAt(i) );
}
}
return result.toString();
}
/**
* This returns the index splitting the static part of the filename from the templated part.
* For example, http://autoplot.org/data/C1_CP_EDI_EGD__$Y$m$d_V$v.cef is split into:<br>
* <pre>http://autoplot.org/data/
*C1_CP_EDI_EGD__$Y$m$d_V$v.cef</pre> and
* http://emfisis.physics.uiowa.edu/Flight/RBSP-A/Quick-Look/$Y/$m/$d/rbsp-a_magnetometer_4sec-gsm_emfisis-Quick-Look_$Y$m$d_v$(v,sep).cdf:<br>
* <pre>http://emfisis.physics.uiowa.edu/Flight/RBSP-A/Quick-Look/
*$Y/$m/$d/rbsp-a_magnetometer_4sec-gsm_emfisis-Quick-Look_$Y$m$d_v$(v,sep).cdf</pre>
* <tt></tt><br>
*
* <p>
* This new version uses regexs and is more complete than versions found in Autoplot, and they should
* eventually use this instead. Note the Autoplot one returns the index of the last /, whereas this
* returns that index plus one.
* </p>
*
* <p>Taken from Autoplot's AggregatingDataSourceFactory, where Autoplot just has a URI and needs to get a file list.
* See also org/autoplot/pngwalk/WalkUtil.java splitIndex, which also allows wildcards like *.</p>
*
* Note, ages ago %Y was valid as well as $Y. This is no longer supported, since it is useful
* to have Python templates working alongside URI_Templates.
*
* @param surl a string like http://autoplot.org/data/C1_CP_EDI_EGD__$Y$m$d_V$v.cef
* @return an integer indicating the split index, so that surl.substring(0,i) includes the slash.
*/
public static int splitIndex(String surl) {
String regex= "([\\$][yYmdjxv\\(\\{])";
Matcher m= Pattern.compile(regex).matcher(surl);
if ( m.find() ) {
int i= m.start();
i = surl.lastIndexOf('/', i);
return i+1;
} else {
return -1;
}
}
/**
* creates a FileStorageModel for the given template, which uses:
* <pre>%Y-%m-%dT%H:%M:%S.%{milli}Z";
* %Y 4-digit year
* %m 2-digit month
* %d 2-digit day of month
* %j 3-digit day of year
* %H 2-digit Hour
* %M 2-digit Minute
* %S 2-digit second
* %v best version by number Also %(v,sep) for 4.3.2 or %(v,alpha)
* %{milli} 3-digit milliseconds </pre>
* product_$(o,id=ftp://stevens.lanl.gov/pub/projects/rbsp/autoplot/orbits/rbspa_pp).png
* @param root FileSystem source of the files.
* @param template describes how filenames are constructed. This is converted to a regular expression and may contain regex elements without groups. The
* string may contain $ instead of percents as long as there are no percents in the string.
*
* @return a newly-created FileStorageModel.
*/
public static FileStorageModel create( FileSystem root, String template ) {
template= makeCanonical(template);
String templatebr= hideParams( template );
int i= templatebr.lastIndexOf("/");
if ( template.contains("%") && !template.contains("$") ) { //TODO: makeCanonical should do this.
template= template.replaceAll("\\%", "\\$");
templatebr= templatebr.replaceAll("\\%", "\\$");
}
int i2= templatebr.lastIndexOf("$",i);
if ( i2 != -1 ) {
String parentTemplate= template.substring(0,i);
FileStorageModel parentFSM= FileStorageModel.create( root, parentTemplate );
return new FileStorageModel( parentFSM, root, template );
} else {
return new FileStorageModel( null, root, template );
}
}
/**
* creates a FileStorageModel for the given template, but with a custom FieldHandler and
* field.
*
* @param root FileSystem source of the files.
* @param template describes how filenames are constructed. This is converted to a regular expression and may contain regex elements without groups. The
* string may contain $ instead of percents as long as there are no percents in the string.
* @param fieldName custom field name
* @param fieldHandler TimeParser.FieldHandler to call with the field contents.
* @return a newly-created FileStorageModel.
*/
public static FileStorageModel create( FileSystem root, String template, String fieldName, TimeParser.FieldHandler fieldHandler ) {
template= makeCanonical(template);
String templatebr= hideParams( template );
int i= templatebr.lastIndexOf("/");
if ( template.contains("%") && !template.contains("$") ) {
template= template.replaceAll("\\%", "\\$");
templatebr= templatebr.replaceAll("\\%", "\\$");
}
int i2= templatebr.lastIndexOf("$",i);
if ( i2 != -1 ) {
String parentTemplate= template.substring(0,i);
FileStorageModel parentFSM= FileStorageModel.create( root, parentTemplate, fieldName, fieldHandler );
return new FileStorageModel( parentFSM, root, template, fieldName, fieldHandler );
} else {
return new FileStorageModel( null, root, template, fieldName, fieldHandler );
}
}
private FileStorageModel( FileStorageModel parent, FileSystem root, String template, String fieldName, TimeParser.FieldHandler fieldHandler, Object ... moreHandler ) {
this.root= root;
this.parent= parent;
if ( template.startsWith(" ") ) {
throw new IllegalArgumentException("template cannot start with space: \""+template+"\"");
}
if ( template.startsWith("/") ) { // clean up double slashes immediately. (/home/jbf//data/)
template= template.substring(1);
}
while ( template.contains("//") ) {
template= template.replaceAll("\\/\\/", "/");
}
this.template= template.replaceAll("\\+", "\\\\+");
String f="v";
versioningType= VersioningType.none;
TimeParser.FieldHandler vh= new TimeParser.FieldHandler() {
@Override
public String configure( Map<String,String> args ) {
String sep= args.get( "sep" );
if ( sep==null && args.containsKey("dotnotation")) {
sep= "T";
}
String alpha= args.get( "alpha" );
if ( alpha==null && args.containsKey("alphanumeric") ) {
alpha="T";
}
String type= args.get("type");
if ( type!=null ) {
if ( type.equals("sep") || type.equals("dotnotation") ) {
sep= "T";
} else if (type.equals("alpha") || type.equals("alphanumeric") ) {
alpha="T";
}
}
if ( args.get("gt")!=null ) {
throw new IllegalArgumentException("gt specified but not supported: must be ge or lt");
}
if ( args.get("le")!=null ) {
throw new IllegalArgumentException("le specified but not supported: must be ge or lt");
}
String ge= args.get("ge");
if ( ge!=null ) {
versionGe= ge;
}
String lt= args.get("lt");
if ( lt!=null ) {
versionLt= lt;
}
if ( alpha!=null ) {
if ( sep!=null ) {
return "alpha with split not supported";
} else {
versioningType= VersioningType.alphanumeric;
}
} else {
if ( sep!=null ) {
versioningType= VersioningType.numericSplit;
} else {
versioningType= VersioningType.numeric;
}
}
return null;
}
@Override
public void parse( String fieldContent, TimeStruct startTime, TimeStruct timeWidth, Map<String,String> extra ) {
String v= extra.get("v");
if ( v!=null ) {
versioningType= VersioningType.numericSplit;
fieldContent= v+"."+fieldContent; // Support $v.$v.$v
}
extra.put( "v", fieldContent );
}
@Override
public String getRegex() {
return ".*";
}
@Override
public String format( TimeStruct startTime, TimeStruct timeWidth, int length, Map<String, String> extra ) {
return extra.get("v"); //TODO: length
}
};
if ( fieldName==null ) {
this.timeParser= TimeParser.create( template, f, vh );
} else {
if ( moreHandler==null || moreHandler.length==0 ) { //TODO: check if it can ever be null.
this.timeParser= TimeParser.create( template, fieldName, fieldHandler, f, vh );
} else {
this.timeParser= TimeParser.create( template, fieldName, fieldHandler, f, vh, moreHandler );
}
}
//if ( this.timeParser.isStartTimeOnly() ) {
// this.startTimeOnly= true;
//}
this.regex= timeParser.getRegex();
this.pattern= Pattern.compile(regex);
if ( template.endsWith(".gz") ) {
allowGz= false; // turn off automatic uncompressing GZ files.
}
if ( allowGz ) {
this.gzpattern= Pattern.compile(regex+"\\.gz");
}
}
// /**
// * The filename time only contains the start time of the interval, the end of the interval
// * is only bounded by the next file.
// */
// protected boolean startTimeOnly = false;
//
// /**
// * limit on the length of files with startTimeOnly set.
// * e.g. $Y$m$d_$(H,startTimeOnly)$M means that the files should be much less that one hour long
// *
// */
// protected Datum implicitTimeDelta= null;
private FileStorageModel( FileStorageModel parent, FileSystem root, String template ) {
this( parent, root, template, null, null );
}
@Override
public String toString() {
return String.valueOf(root) + template;
}
}