/* * To change this template, choose Tools | Templates * and open the template in the editor. */ package org.autoplot.idlsupport; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.text.NumberFormat; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.logging.Level; import java.util.logging.LogManager; import java.util.logging.Logger; import org.autoplot.datasource.AutoplotSettings; import org.das2.qds.buffer.FloatDataSet; import org.das2.qds.buffer.LongDataSet; import org.das2.datum.LoggerManager; import org.das2.datum.Units; import org.das2.datum.UnitsConverter; import org.das2.util.monitor.NullProgressMonitor; import org.das2.util.monitor.ProgressMonitor; import org.das2.qds.DataSetOps; import org.das2.qds.DataSetUtil; import org.das2.qds.FDataSet; import org.das2.qds.QDataSet; import org.das2.qds.QubeDataSetIterator; import org.das2.qds.SemanticOps; /** * See http://autoplot.org/IDL and http://autoplot.org/Matlab which show how this is used in the environments. * @author jbf */ public abstract class QDataSetBridge { private static final Logger logger= LoggerManager.getLogger("qdataset.bridge"); QDataSet ds; Exception exception; String name; Map datasets; /** * this contains the names of things that should resolve when slicing. */ Map sliceDep; Map names; List prefUnits; // convert to these if possible double fill; // use this fill value float ffill; long lfill= -999999999999999999L; // hope this isn't used. boolean useFill=false; // true means convert fill values boolean debug= false; QDataSetBridge() { datasets = new LinkedHashMap<>(); names = new LinkedHashMap<>(); sliceDep= new LinkedHashMap<>(); prefUnits= new ArrayList(); } /** * set the preferred units for the data. The code will convert * to these units when a converter is found. * If a preference is indicated when it is convertible to an existing * preference, the existing preference is removed. * See clearPreferredUnits to remove all preferences. * Example units strings (capitalization matters): **
{@code
     *    seconds since 2010-01-01T00:00
     *    days since 2010-01-01T00:00
     *    Hz
     *    kHz
     *    MHz
     *}
* @param sunit */ public void setPreferredUnits( String sunit ) { Units unit; if ( sunit.contains(" since ") ) { unit= Units.lookupUnits(sunit); } else { unit= Units.getByName(sunit); } boolean add= true; for ( int i=0; i apds.setFillValue, 89. * % setFillValue(float) * IDL> apds.setFillValue, 89.d * % setFillValue(double) * @param d */ public void setFillValue( double d ) { setFillDouble( d ); } public void setFillValue( float f ) { this.ffill= f; this.fill= (double)f; // danger this.useFill= true; } /** * set the value to return when the data is invalid, when the data is known to be stored as doubles * (8-byte numbers). Python JPype doesn't allow for operator overloading, so this should be used. * @param d */ public void setFillDouble( double d ) { this.fill= d; this.ffill= (float)d; // danger this.useFill= true; } protected String filter = ""; public static final String PROP_FILTER = "filter"; /** * get the filter that is applied to the data immediately after it is loaded. * @return the filter string, empty string means no filters. */ public String getFilter() { return filter; } /** * set the filter that is applied to the data immediately after it is loaded. * @param filter filter string like "|histogram()" or "" for no filters. */ public void setFilter(String filter) { this.filter = filter; } /** * turn on/off debug messages * @param debug */ public void setDebug( boolean debug ) { System.err.println("setting debug="+debug); this.debug= debug; } /** * don't use fill value. */ public void clearFillValue() { this.useFill= false; } /** * performs the read. Note no progress status is available and this blocks until the read is done. * doGetDataSet(mon) should be called if progress is needed. * 2011-01-01: getStatus or getStatusMessage should be called afterwards to check the result of the load, this will no longer throw an exception. */ public void doGetDataSet() { this.exception= null; try { this.ds = getDataSet( new NullProgressMonitor() ); if ( this.filter.length()>0 ) { ds= DataSetOps.sprocess( this.filter, ds, new NullProgressMonitor() ); } datasets.clear(); name = nameFor(ds); datasets.put(name, ds); if ( SemanticOps.isBundle(ds) ) { String[] ss= DataSetOps.bundleNames(ds); for ( String s: ss ) { datasets.put( s, DataSetOps.unbundle(ds,s) ); } } for (int i = 0; i < ds.rank(); i++) { QDataSet dep = (QDataSet) ds.property("DEPEND_" + i); if (dep != null) datasets.put(nameFor(dep), dep); QDataSet depslice= (QDataSet) ds.property("DEPEND_" + i, 0 ); if ( depslice!=null ) { sliceDep.put( nameFor(depslice,true), "DEPEND_"+i ); } } QDataSet ads; int i; i=0; while ( (ads=(QDataSet)ds.property( "PLANE_"+i ))!=null ) { datasets.put( nameFor( ads ), ads ); i++; } i=0; while ( (ads=(QDataSet)ds.property( "PLANE_"+i, 0 ))!=null ) { sliceDep.put( nameFor( ads,true ), "PLANE_"+i ); i++; } } catch ( Exception ex ) { this.exception= ex; logger.log(Level.WARNING, null, ex ); } } /** * This initiates the read on a separate thread, so this does not * block and should be used with caution. See getProgressMonitor for * use. Note this is more advanced and is intended to support use in * other software (e.g. PaPCo). * * Note because there is one exception that is stored, a QDataSetBridge object * is only able to load one dataset at a time. Simultaneous loads should * be done with multiple QDataSetBridge objects. * * @param mon progress monitor from getProgressMonitor() */ public void doGetDataSet(final ProgressMonitor mon) { this.exception= null; Runnable run = new Runnable() { @Override public void run() { datasets.clear(); name= ""; try { ds = getDataSet(mon); if ( ds==null ) return; if ( filter.length()>0 ) { ds= DataSetOps.sprocess( filter, ds, new NullProgressMonitor() ); } } catch (Exception ex) { exception= ex; logger.log( Level.WARNING, null, ex ); mon.setProgressMessage("EXCEPTION"); mon.finished(); return; } name = nameFor(ds); datasets.put(name, ds); for (int i = 0; i < ds.rank(); i++) { QDataSet dep = (QDataSet) ds.property("DEPEND_" + i); if (dep != null) datasets.put(nameFor(dep), dep); QDataSet depslice= (QDataSet) ds.property("DEPEND_" + i, 0 ); if ( depslice!=null ) { sliceDep.put( nameFor(depslice), "DEPEND_"+i ); } } } }; new Thread(run).start(); } /** * return the Exception from the last doGetDataSet call. * @return */ public Exception getException() { return exception; } /** * returns 0 for last get operation successful * @return */ public int getStatus() { return exception==null ? 0 : 1; } /** * returns "" for last operation successful, or non-empty indicating the problem. Note * getException will return the Java exception for deeper inspection. * @return "" or the error message */ public String getStatusMessage() { if ( exception==null ) { return ""; } else { String s= exception.getMessage(); if ( s!=null && s.length()>0 ) { return s; } else { return exception.toString(); // sorry! } } } /** * return the name used to refer to the dataset. * This may add to the list of datasets. * @param dep0 the dataset * @return the name for the dataset. */ public synchronized String nameFor(QDataSet dep0) { return nameFor( dep0, false ); } /** * return the name used to refer to the dataset. * This may add to the list of datasets. * @param dep0 the dataset * @return the name for the dataset. */ private synchronized String nameFor(QDataSet dep0, boolean onlySlice ) { String name1 = names.get(dep0); if ( name1==null ) { name1 = (String) dep0.property(QDataSet.NAME); if ( name1 == null || datasets.containsKey(name1) ) { name1 = "ds_" + names.size(); } names.put(dep0, name1); if ( datasets.containsKey(name1) ) { throw new IllegalArgumentException("dataset name is already taken: "+name1 ); } if ( !onlySlice ) datasets.put( name1, dep0 ); } return name1; } /** * implementations should provide this method for making the data accessible. * @param mon * @return * @throws Exception if any problem occurs with the read, it will be available to clients in getException or getStatusMessage */ abstract QDataSet getDataSet(ProgressMonitor mon) throws Exception; /** * returns an object that can be used to monitor the progress of a download. * * mon= qds->getProgressMonitor(); * qds->doGetDataSet( mon ) * while ( ! mon->isFinished() ) do begin * print, strtrim( mon->getTaskProgress(), 2 ) + " " + strtrim( mon->getTaskSize(), 2 ) * wait, 0.2 ; don't overload the thread * endwhile * * @return */ public ProgressMonitor getProgressMonitor() { return new NullProgressMonitor(); } public void values(String name, double[] result) { if ( debug ) { System.err.println("reading "+name+" into double["+result.length+"]" ); } QDataSet ds1 = datasets.get(name); copyValues( ds1, result ); } public void svalues( String name, String[] result) { if ( debug ) { System.err.println("reading "+name+" into double["+result.length+"]" ); } QDataSet ds1 = datasets.get(name); if (ds1==null ) { throw new IllegalArgumentException("no dataset with the name \""+name+"\"" ); } for ( int i=0; i properties(String name,int i) { LinkedHashMap result = new LinkedHashMap<>(DataSetUtil.getProperties(datasets.get(name).slice(i))); //TODO: strange implementation for (String s : result.keySet()) { result.put(s, property(name, s,i)); } return result; } /** * returns one of String, int, double, float, int[], double, float[] * @param name * @param propname * @return */ public Object property(String name, String propname) { Object prop = datasets.get(name).property(propname); if (prop instanceof QDataSet) { return nameFor((QDataSet) prop); } else if (prop instanceof Units) { Units dsu= SemanticOps.getUnits( datasets.get(name) ); for ( Units u: prefUnits ) { if ( u.isConvertibleTo( dsu ) ) { return u.toString(); } } return prop.toString(); } else if ( propname.equals(QDataSet.FILL_VALUE) && this.useFill ) { return this.fill; } else { return prop; } } public boolean hasProperty(String name, String propname) { return datasets.get(name).property(propname) != null; } /** * get the properties for the named dataset * @param name * @return */ public Map properties(String name) { LinkedHashMap result = new LinkedHashMap<>(DataSetUtil.getProperties(datasets.get(name))); for (String s : result.keySet()) { result.put(s, property(name, s)); } return result; } /** * returns one of String, int, double, float, int[], double, float[] * @param propname * @return */ public Object property(String propname) { Object prop = datasets.get(name).property(propname); if (prop instanceof QDataSet) { return nameFor((QDataSet) prop); } else if (prop instanceof Units) { return prop.toString(); } else { return prop; } } public boolean hasProperty(String propname) { return datasets.get(name).property(propname) != null; } public Map properties() { LinkedHashMap result = new LinkedHashMap<>(DataSetUtil.getProperties(datasets.get(name))); for (String s : result.keySet()) { result.put(s, property(name, s)); } return result; } /** * returns one of String, int, double, float, int[], double, float[] * @param propname the property name * @param i the slice index * @return the name of the qdataset, or the value. */ public Object property(String propname, int i) { Object prop = datasets.get(name).property(propname,i); if (prop instanceof QDataSet) { return nameFor((QDataSet) prop); } else if (prop instanceof Units) { return prop.toString(); } else { return prop; } } public boolean hasProperty(String propname,int i) { return datasets.get(name).property(propname,i) != null; } public Map properties(int i) { LinkedHashMap result = new LinkedHashMap(DataSetUtil.getProperties(datasets.get(name).slice(i) )); return result; } public String[] names() { return datasets.keySet().toArray(new String[datasets.size()]); } public String name() { return name; } /** * print the Java memory stats to stderr. * @see #clearMemory() */ public void reportMemory() { System.err.println( "= Java Runtime Information =" ); String javaVersion= System.getProperty("java.version"); String javaVersionWarning= ""; // The java about checks for 1.8.102 String arch = System.getProperty("os.arch");// applet okay NumberFormat nf= new java.text.DecimalFormat(); String mem = nf.format( (Runtime.getRuntime()).maxMemory() / 1000000 ); String tmem= nf.format( (Runtime.getRuntime()).totalMemory() / 1000000 ); String fmem= nf.format( (Runtime.getRuntime()).freeMemory() / 1000000 ); System.err.println( "Java version: " + javaVersion + " " + javaVersionWarning ); System.err.println( "Arch: " + arch ); System.err.println( "Max memory (MB): " + mem + " (memory available to process)" ); System.err.println( "Total memory (MB): " + tmem + " (amount allocated to the process)" ); System.err.println( "Free memory (MB): " + fmem + " (amount available before more must be allocated)" ); } /** * return the total memory and free memory available to the Java process, * in megabytes (1e6 bytes). * @return an integer array [ used, total ] */ public int[] freeMemory() { int mem = (int)( (Runtime.getRuntime()).maxMemory() / 1000000 ); int tmem= (int)( (Runtime.getRuntime()).totalMemory() / 1000000 ); int fmem= (int)( (Runtime.getRuntime()).freeMemory() / 1000000 ); return new int[] { tmem-fmem, mem }; } /** * clear existing data from memory, in case the bridge object is not cleared * from in IDL or Matlab memory. * @see #reportMemory() */ public void clearMemory() { datasets.clear(); } /** * load the configuration from autoplot_data/config/logging.properties */ public void readLogConfiguration() { // read in the file $HOME/autoplot_data/config/logging.properties, if it exists. File f1= new File( AutoplotSettings.settings().resolveProperty( AutoplotSettings.PROP_AUTOPLOTDATA ), "config" ); File f2= new File( f1, "logging.properties" ); if ( f2.exists() ) { if ( !f2.canRead() ) logger.log(Level.WARNING, "Unable to read {0}", f2); InputStream in=null; try { logger.log(Level.INFO, "Reading {0}", f2); in= new FileInputStream(f2); LogManager.getLogManager().readConfiguration(in); } catch ( IOException ex ) { logger.log(Level.WARNING, "IOException during read of {0}", f2); } finally { try { if ( in!=null ) in.close(); } catch ( IOException ex ) { logger.log(Level.WARNING, "IOException during close of {0}", f2); } } } } /** * desperate method for debugging where JPype/Python would hang, in hopes that * this might show where it's hanging. * @deprecated this is no longer a problem and will be removed */ public void dumpStack() { Map mm= Thread.getAllStackTraces(); for ( Entry t: mm.entrySet() ) { System.err.println("Thread: "+t.getKey().getName()); for ( StackTraceElement st: t.getValue() ) { System.err.println(" "+st.toString() ); } System.err.println(""); } } /** * desperate method for debugging where JPype/Python would hang, in hopes that * this might show where it's hanging. * @param n number of seconds. * @deprecated this is no longer a problem and will be removed */ public void dumpStackInNSeconds( final double n ) { Runnable run= new Runnable() { @Override public void run() { try { Thread.sleep( (int)( n * 1000 ) ); } catch (InterruptedException ex) { logger.log(Level.SEVERE, null, ex); } dumpStack(); } }; new Thread(run).start(); } }