/* * To change this template, choose Tools | Templates * and open the template in the editor. */ package org.das2.qds; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import org.das2.datum.InconvertibleUnitsException; import org.das2.datum.LoggerManager; import org.das2.datum.Units; import org.das2.datum.UnitsConverter; import org.das2.util.monitor.ProgressMonitor; import org.das2.qds.ops.Ops; /** * DataSetIterator implementation that can be used for all dataset (not just qubes). * Originally this only worked for QDataSets that were qubes, or datasets that * had the same dataset geometry for each slice. At some point this was * modified to work with any dataset but the name remains. * * DataSetIterators are intended to work with multiple datasets at once. For example, * if we want to add the data from two datasets together, we would create one * iterator that would be used to access both datasets. One dataset is provided * to the constructor, but any dataset of the same geometry can be passed to the * getValue method. * * TODO: This does not work for Rank 0 datasets. See * sftp://klunk.physics.uiowa.edu/home/jbf/project/autoplot/script/demos/jeremy/qubeDataSetIteratorForNonQubes.jy * * @author jbf */ public final class QubeDataSetIterator implements DataSetIterator { private static final Logger logger= LoggerManager.getLogger("qdataset.iterator"); /** * DimensionIterator iterates over an index. For example, using * Jython for brevity: *
{@code
     * ds= zeros(15,4,2)
     * ds[:,:,:] has itertors that count of 0,...,14; 0,...,3; and 0,1
     * ds[3:15,:,:]  uses a StartStopStepIterator to count off 3,4,5,...,14
     * ds[3,:,:]  uses a SingletonIterator
     * i1= [0,1,2,3]
     * i2= [0,0,1,1]
     * i3= [0,1,0,1]
     * ds[i1,i2,i3]  # uses IndexListIterator
     *}
* When index is called before hasNext, it must return -1 to indicate an uninitialized state. */ public interface DimensionIterator { /** * true if there are more indices in the iteration * @return true if there are more indices in the iteration */ boolean hasNext(); /** * return the next index of the iteration * @return the next index of the iteration */ int nextIndex(); /** * return the current index. * @return the current index. */ int index(); /** * return the length of the iteration. * @return the length of the iteration. */ int length(); } /** * DimensionIteratorFactory creates DimensionIterators */ public interface DimensionIteratorFactory { DimensionIterator newIterator(int len); } /** * Iterator for counting off indices. (3:15:2 in ds[3:15:2,:]) */ public static class StartStopStepIterator implements DimensionIterator { int start; int stop; int step; int index; boolean all; // just for toString public StartStopStepIterator(int start, int stop, int step, boolean all) { this.start = start; this.stop = stop; this.step = step; this.index = start - step; this.all = all; } @Override public boolean hasNext() { if ( step>=0 ) { return index + step < stop; } else { return index + step > stop; } } @Override public int nextIndex() { index += step; return index; } @Override public int index() { return index; } @Override public int length() { int remainder= (stop - start) % step; return (stop - start) / step + ( remainder>0 ? 1 :0 ); } @Override public String toString() { return all ? ":" : "" + start + ":" + stop + (step == 1 ? "" : ":" + step); } } /** * generates iterator for counting off indices. (3:15:2 in ds[3:15:2,:]) * Indices can be negative. */ public static class StartStopStepIteratorFactory implements DimensionIteratorFactory { Number start; Number stop; Number step; /** * create the factory which will create iterators. * @param start the start index. negative indices are supported. * @param stop the stop index, exclusive. null (or None) is used to indicate the end of non-qube datasets. * @param step the step size, null means just use 1. */ public StartStopStepIteratorFactory(Number start, Number stop, Number step) { this.start = start; this.stop = stop; this.step = step; } @Override public DimensionIterator newIterator(int length) { int step1 = step == null ? 1 : step.intValue(); int dftStart,dftStop; if ( step1>=0 ) { dftStart= 0; dftStop= length; } else { dftStart= -1; dftStop= -1-length; // note danger code which assumes this will still be negative below. } int start1 = start == null ? dftStart : start.intValue(); int stop1 = stop == null ? dftStop : stop.intValue(); if (start1 < 0) { start1 = length + start1; } if (stop1 < 0) { stop1 = length + stop1; } return new StartStopStepIterator(start1, stop1, step1, start == null && stop == null && step == null); } } /** * Iterator that goes through a list of indices. */ public static class IndexListIterator implements DimensionIterator { QDataSet ds; int listIndex; public IndexListIterator(QDataSet ds) { if ( ds.rank()==0 ) { ds= Ops.join(null,ds); } this.ds = ds; if ( ds.rank()!=1 ) { throw new IllegalArgumentException("list of indices dataset must be rank 1"); } this.listIndex = -1; } @Override public boolean hasNext() { return listIndex+1 < ds.length(); } @Override public int nextIndex() { listIndex++; return (int) ds.value(listIndex); } @Override public int index() { return (int) ds.value(listIndex); } @Override public int length() { return ds.length(); } @Override public String toString() { String dstr= ds.toString(); dstr= dstr.replace("(dimensionless)", ""); return "[" + dstr + " @ " +listIndex + "]"; } } /** * return the current line in the Jython script as <filename>:<linenum> * or ??? if this cannot be done. Note calls to this will collect a stack * trace and will affect performance. * @return the current line or ??? * @see JythonOps#currentLine() */ public static String currentJythonLine() { StackTraceElement[] sts= new Exception().getStackTrace(); int i= 0; while ( i-1 ) { Number max= (Number)ds.property(QDataSet.VALID_MAX); if ( max!=null ) { if ( max.intValue()!=indexSize ) { //&& (indexSize-max.intValue()!=1) String jythonLine= currentJythonLine(); if ( jythonLine.equals("???") ) { if ( max instanceof Integer || max instanceof Long ) { logger.log(Level.WARNING, "rfe737: index list appears to be for dimensions of length {0} (see VALID_MAX) but is indexing dimension length {1}, which may indicate there''s a bug.", new Object[]{max, indexSize }); } else { if ( max.doubleValue()>1e100 ) { } else { logger.log(Level.WARNING, "rfe737: VALID_MAX is not an integer but data is used as an index list, which may indicate there''s a bug.", new Object[]{max, indexSize }); } } } else { logger.log(Level.WARNING, "rfe737: index list appears to be for dimensions of length {0} (see VALID_MAX) but is indexing dimension length {1}, which may indicate there''s a bug at {2}.", new Object[]{max, indexSize, jythonLine}); } } } } // check the first hundred indices to see if they are non-integer. int n= Math.min( ds.length(), 100 ); for ( int i=0; i problems= new ArrayList<>(); if ( ! DataSetUtil.validate(ds,problems) ) { throw new IllegalArgumentException("data doesn't validate: "+problems ); } this.rank = ds.rank(); it= new DimensionIterator[ this.rank ]; fit= new DimensionIteratorFactory[ this.rank ]; if ( DataSetUtil.isQube(ds) ) { this.qube = DataSetUtil.qubeDims(ds); this.ds = ds; } else { this.qube= null; this.ds = ds; } for (int i = 0; i < this.rank; i++) { fit[i] = new StartStopStepIteratorFactory(0, null, 1); } initialize(); } /** * internal constructor. Note we have to have both constructors, which must * be kept in sync, because of initialize(). * @param ds * @param fits */ private QubeDataSetIterator(QDataSet ds, DimensionIteratorFactory[] fits) { if (Boolean.TRUE.equals(ds.property(QDataSet.QUBE))) { //TODO: why is this different? this.qube = DataSetUtil.qubeDims(ds); this.ds = ds; } else { this.qube= null; this.ds = ds; } this.rank = ds.rank(); this.ds = ds; this.fit = fits; it= new DimensionIterator[ fits.length ]; initialize(); } /** * convenient method for monitoring long processes, this allows * clients to let the iterator manage the monitor. setTaskSize, * started, setTaskProgress, and finished are called automatically. * The monitor will reflect the zeroth index. * @param mon the monitor, or null. */ public void setMonitor( ProgressMonitor mon ) { this.monitor= mon; this.monitor.setTaskSize( this.rank==0 ? 1 : this.dimLength(0) ); this.monitor.started(); } /** * return an iterator for the slice of a dataset (on the 0th index). * This is introduced to improve performance by reducing the number of * bounds checks, etc from the general case. Note slice is a native * operation for most datasets now, so this is probably obsolete. * @param ds the dataset. * @param sliceIndex the index of the slice. * @return an iterator for the slice. * @see org.das2.qds.QDataSet#slice(int) */ public static QubeDataSetIterator sliceIterator( QDataSet ds, int sliceIndex ) { QubeDataSetIterator result; switch (ds.rank()) { case 0: throw new IllegalArgumentException("can't slice rank 0"); case 1: throw new IllegalArgumentException("can't slice rank 1"); case 2: result = new QubeDataSetIterator(ds, new DimensionIteratorFactory[]{new SingletonIteratorFactory(sliceIndex), new StartStopStepIteratorFactory(0, ds.length(sliceIndex), 1)}); break; case 3: result = new QubeDataSetIterator(ds, new DimensionIteratorFactory[]{new SingletonIteratorFactory(sliceIndex), new StartStopStepIteratorFactory(0, ds.length(sliceIndex), 1), new StartStopStepIteratorFactory(0, null, 1),}); break; case 4: result = new QubeDataSetIterator( ds, new DimensionIteratorFactory[]{new SingletonIteratorFactory(sliceIndex), new StartStopStepIteratorFactory(0, ds.length(sliceIndex), 1), new StartStopStepIteratorFactory(0, null, 1), new StartStopStepIteratorFactory(0, null, 1), } ); break; default: throw new IllegalArgumentException("rank limit"); } return result; } /** * reinitializes the iterator. * @param dim the dimension index (0,...,rank-1) * @param fit the iterator factory to use for the index. */ public void setIndexIteratorFactory(int dim, DimensionIteratorFactory fit) { if ( dim >= this.rank ) { throw new IllegalArgumentException( String.format( "rank limit: rank %d dataset %s has no index %d", ds.rank(), ds, dim ) ); } if ( fit instanceof IndexListIteratorFactory ) { IndexListIteratorFactory ffit= (IndexListIteratorFactory)fit; int max= ffit.getMaxIndexExclusive(); if ( max>-1 ) { if ( this.qube!=null ) { if ( this.qube[dim]!=max ) { if ( this.qube[dim]-max==1 ) { logger.log(Level.FINER, "rfe737: index list appears to be for dimensions of length {0} (see VALID_MAX) but is indexing dimension length {1}, I bet you know what you are doing...", new Object[]{max, this.qube[dim]} ); } else { String jythonLine= currentJythonLine(); if ( jythonLine.equals("???") ) { logger.log(Level.WARNING, "rfe737: index list appears to be for dimensions of length {0} (see VALID_MAX) but is indexing dimension length {1}, which may indicate there''s a bug.", new Object[]{max, this.qube[dim]}); } else { logger.log(Level.WARNING, "rfe737: index list appears to be for dimensions of length {0} (see VALID_MAX) but is indexing dimension length {1}, which may indicate there''s a bug at {2}.", new Object[]{max, this.qube[dim], jythonLine}); } } } } } } this.fit[dim] = fit; initialize(); } /** * now that the factories are configured, initialize the iterator to * begin iterating. */ private void initialize() { boolean allLi= true; boolean zeroLength= false; initialNext= new boolean[rank]; if ( rank>0 ) initialNext[rank-1]= true; for (int i = 0; i < rank; i++) { initialNext[i]= true; int dimLength= dimLength(i); if ( dimLength==0 ) zeroLength= true; it[i] = fit[i].newIterator(dimLength); if ( !( it[i] instanceof IndexListIterator ) ) allLi= false; if ( i>0 && i 0) { for (int j = i - 1; j >= 0; j--) { if (it[j].hasNext()) { if ( qube!=null ) { return true; } else { if ( j==0 ) { int nslice; int nextIndex= it[j].index(); do { nextIndex++; QDataSet sliceDs= ds.slice(nextIndex); // The slice of any QDataSet is a qube nslice= DataSetUtil.product( DataSetUtil.qubeDims(sliceDs) ); } while ( nslice==0 && nextIndex0; } else { return true; } } } } } } if ( monitor!=null ) monitor.finished(); return false; } } /** * advance to the next index. getValue(ds) will return the value of the * dataset at this index. * @see #getValue(org.das2.qds.QDataSet) */ @Override public void next() { if (this.allnext) { for (int i = 0; i < (rank - 1); i++) { if ( initialNext[i] ) { it[i].nextIndex(); initialNext[i]= false; } if ( i==0 && monitor!=null ) monitor.setTaskProgress(it[0].index()); } this.allnext = false; // This will never be true again if (rank == 0) { return; } } // implement borrow logic int i = rank - 1; if (it[i].hasNext()) { // typical case, where we have more elements in the last index. it[i].nextIndex(); if ( it[i] instanceof IndexListIterator ) { // all index lists need to be incremented together. for ( int k=0; k0) { for (int j = i - 1; j >= 0; j--) { if (it[j].hasNext()) { it[j].nextIndex(); if ( j==0 && monitor!=null ) { monitor.setTaskProgress(it[0].index()); } if ( it[j] instanceof IndexListIterator ) { // all index lists need to be incremented together. for ( int k=0; k qqube = new ArrayList<>(); List dimMap= new ArrayList<>(); //e.g. [0,1,2,3] for (int i = 0; i < this.it.length; i++) { if ( this.it[i]==null ) { continue; } boolean reform= this.it[i] instanceof SingletonIterator; if (!reform) { qqube.add( this.it[i].length() ); dimMap.add( i ); } } int[] qube1; if (isAllIndexLists) { // a=[1,2,3]; b=[2,3,1]; Z[a,b] is rank 1 int len= this.it[0].length(); for ( int i=1; i