package org.autoplot.idlsupport; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.lang.reflect.Array; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.logging.Level; import java.util.logging.Logger; /** * Read data from IDL Save Files. This was written using * http://www.physics.wisc.edu/~craigm/idl/savefmt/node20.html * https://cow.physics.wisc.edu/~craigm/idl/savefmt.pdf * and https://github.com/scipy/scipy/blob/master/scipy/io/idl.py * for reference, and with no involvement from individuals at * Harris Geospacial. No warranties are implied and this must * be used at your own risk. * *
{@code
 * from  org.autoplot.idlsupport import ReadIDLSav
 * reader= ReadIDLSav()
 * aFile= File('/tmp/aDataFile.sav')
 * inChannel = aFile.getChannel
 * fileSize = inChannel.size()
 * buffer = ByteBuffer.allocate( inChannel.size() )
 * bytesRead= 0;
 * while ( bytesRead
 * @author jbf
 */
public class ReadIDLSav {
        
    private static final Logger logger= Logger.getLogger("apdss.idlsav");
            
    private static final int RECTYPE_VARIABLE = 2;
    private static final int RECTYPE_ENDMARKER = 6;
    private static final int RECTYPE_TIMESTAMP = 10;
    private static final int RECTYPE_VERSION = 14;
    private static final int RECTYPE_PROMOTE64 = 17;
    
    private static final int VARFLAG_ARRAY = 0x04;
    private static final int VARFLAG_STRUCT = 0x20;
    
    /**
     * return the next record buffer, or returns null at the end.
     * @param ch the bytebuffer
     * @param pos the position.
     * @return the record, including the twelve bytes at the beginning
     * @throws IOException 
     */    
    private ByteBuffer readRecord( ByteBuffer ch, int pos ) throws IOException {

        ch.order( ByteOrder.BIG_ENDIAN );
        
        int type= ch.getInt(pos);
        int endpos= ch.getInt(pos+4);
        
        String stype;
        if ( type==RECTYPE_ENDMARKER ) {
            return null;
        } else {
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    stype= "variable";
                    StringData varName= readString( ch, pos+20 );
                    return slice(ch, pos, endpos, stype, varName.string );
                case RECTYPE_VERSION:
                    stype= "version";
                    break;
                case RECTYPE_TIMESTAMP:
                    stype="timestamp";
                    break;
                case RECTYPE_PROMOTE64:
                    stype="promote64";
                    break;
                default:
                    stype="???";
                    break;
            }
            return slice(ch, pos, endpos, stype, "" );
        }
    }
    
    /**
     * return the next record buffer, or returns null at the end.
     * @param inch the file channel
     * @param pos the position.
     * @return the record, including the twelve bytes at the beginning
     * @throws IOException 
     */    
    private ByteBuffer readRecord( FileChannel inch, int pos ) throws IOException {

        ByteBuffer b8= ByteBuffer.allocate(8);
        inch.read(b8,pos);
        b8.order( ByteOrder.BIG_ENDIAN );
        
        int type= b8.getInt(0);
        int endpos= b8.getInt(4);
                
        String stype;
        if ( type==RECTYPE_ENDMARKER ) {
            return null;
        } else {
            ByteBuffer ch1= ByteBuffer.allocateDirect(endpos-pos);
            inch.read(ch1,pos);
        
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    stype= "variable";
                    StringData varName= readString( ch1, 20 );
                    return sliceLabel( ch1, pos, endpos, stype, varName.string );
                case RECTYPE_VERSION:
                    stype= "version";
                    break;
                case RECTYPE_TIMESTAMP:
                    stype="timestamp";
                    break;
                case RECTYPE_PROMOTE64:
                    stype="promote64";
                    break;
                default:
                    stype="???";
                    break;
            }
            return ch1;
        }
    }
    
    /**
     * somehow I didn't notice the length before other strings.  In the Python 
     * code they have "_read_string" and "_read_string_data" which has a 
     * second length.
     * @param rec
     * @param pos
     * @return StringDesc to describe the string.
     */
    private StringData readStringData( ByteBuffer rec, int pos ) {
        int len= rec.getInt(pos);
        byte[] mybytes= new byte[len];
        rec.position(pos+4);
        rec.get(mybytes);
        StringData result= new StringData();
        result.string= new String( mybytes );
        result._lengthBytes= 4 + Math.max( 4, (int)( 4 * Math.ceil( ( len ) / 4.0 ) ) ); 
        return result;
    }
    
    
    private StringData readString( ByteBuffer rec, int pos ) {
        int endPos= pos;
        while ( rec.get(endPos)!=0 ) {
            endPos++;
        }
        byte[] mybytes= new byte[endPos-pos];
        rec.position(pos);
        rec.get(mybytes);
        StringData result= new StringData();
        result.string= new String( mybytes );
        result._lengthBytes= Math.max( 4, (int)( 4 * Math.ceil( ( result.string.length() ) / 4.0 ) ) ); 
        return result;
    }
    
    public static final int TYPECODE_COMPLEX_FLOAT_SCALAR=0;
    public static final int TYPECODE_BYTE=1;
    public static final int TYPECODE_INT16=2;
    public static final int TYPECODE_INT32=3;
    public static final int TYPECODE_FLOAT=4;
    public static final int TYPECODE_DOUBLE=5;
    public static final int TYPECODE_COMPLEX_FLOAT=6;
    public static final int TYPECODE_STRING=7;
    public static final int TYPECODE_STRUCT=8;
    public static final int TYPECODE_COMPLEX_DOUBLE=9;
    public static final int TYPECODE_INT64=14;
    public static final int TYPECODE_UINT64=15;

    /**
     * return a string representing the type code, if supported.
     * @param typeCode for example 4 which means float or 7 which means string.
     * @return "float" or "string" or whatever the code is, or the numeric code if not supported.
     */
    public static String decodeTypeCode( int typeCode ) {
        switch ( typeCode ) {
            case TYPECODE_BYTE: {
                return "byte";
            }
            case TYPECODE_INT16: {
                return "short";
            }
            case TYPECODE_INT32: {
                return "int";
            }
            case TYPECODE_INT64: {
                return "long";
            }
            case TYPECODE_FLOAT: {
                return "float";
            }   
            case TYPECODE_DOUBLE: {
                return "double";
            }
            case TYPECODE_COMPLEX_DOUBLE: {
                return "complex_double";
            }            
            case TYPECODE_COMPLEX_FLOAT: {
                return "complex_float";
            }            
            case TYPECODE_STRUCT: {
                return "struct";
            }            
            case TYPECODE_STRING: {
                return "string";
            }
            default:
                return String.valueOf(typeCode);
        }
    }
    
    /**
     * return the size of the IDL data type in bytes.  Note shorts are stored
     * in 4-bytes.
     * @param typeCode
     * @return 
     */
    private static int sizeOf( int typeCode ) {
        int[] sizes= new int[] { 0, 4, 4, 4, 4,   8, 16, 1, 0, 32,   0, 0, 0, 0, 8, 8 };
        return sizes[typeCode];
    }
    
    /**
     * read the TypeDesc for the variable.
     * @param in
     * @param name
     * @return
     * @throws IOException 
     */
    private TypeDesc readTypeDesc( ByteBuffer in, String name ) throws IOException {
        int magic= in.getInt(0);
        if ( magic!=1397882884 ) {
            logger.warning("magic number is incorrect");
        }
        int pos= 4;
        
        ByteBuffer rec= readRecord( in, pos );
        while ( rec!=null ) {
            int type= rec.getInt(0);
            int nextPos= rec.getInt(4);
            logger.log(Level.CONFIG, "RecType: {0} Length: {1,number,#}", new Object[]{labelType(type), nextPos-pos});
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    logger.config("variable");
                    StringData varName= readString( rec, 20 );
                    if ( name.startsWith(varName.string) ) {
                        int nextField= 20 + varName._lengthBytes;
                        ByteBuffer var= slice(rec, nextField, rec.limit(), "variablestruct", name );
                        TypeDesc td= readTypeDesc(var);
                        return td; // struct
                    } else if ( varName.string.equals(name) ) {
                        int nextField= 20 + varName._lengthBytes;
                        ByteBuffer var= slice(rec, nextField, rec.limit(), "variable", name );
                        TypeDesc td= readTypeDesc(var);
                        return td;
                    }
                    break;
                case RECTYPE_VERSION:
                    logger.config("version");
                    break;
                case RECTYPE_TIMESTAMP:
                    logger.config("timestamp");
                    break;
                default:
                    logger.config("???");
                    break;
            }
            pos= nextPos;
            rec= readRecord( in, pos );
        }
        throw new IllegalArgumentException("unable to find variable: "+name);        
    }
    /**
     * return true if the name refers to an array
     * @param in ByteBuffer for the entire file
     * @param name the variable name
     * @return td.isStructure();
     */
    public boolean isArray(ByteBuffer in, String name) throws IOException {
        TypeDesc td= readTypeDesc(in, name);
        return isArray( td.varFlags );
    }
    
    /**
     * return true if the name refers to a structure
     * @param in ByteBuffer for the entire file
     * @param name the variable name
     * @return true if the name refers to a structure
     */
    public boolean isStructure(ByteBuffer in, String name) throws IOException {
        TypeDesc td= readTypeDesc(in, name);
        return isStructure( td.varFlags );
    }

    private TagDesc findStructureTag(StructDesc structDesc, String s) {
        String[] ss= s.split("\\.",2);
        int istruct= 0;
        int iarray= 0;
        if ( ss.length==1 ) {
            int itagfind=-1;
            for ( int itag=0; itag1024 ) {
                        throw new IllegalArgumentException("unbelievable len, something has gone wrong.");
                    }
                    byte[] bb= new byte[len];
                    for ( int i=0; i1024 ) {
                            throw new IllegalArgumentException("string has unbelievable len, something has gone wrong.");
                        }
                        byte[] bb= new byte[len];
                        for ( int k=0; k accumulator, Map rec, int j, int nj ) {
        if ( accumulator.entrySet().isEmpty() ) {
            for ( Entry e: rec.entrySet() ) {
                Object o;
                if ( e.getValue() instanceof ArrayData ) {
                    ArrayData ad= (ArrayData)e.getValue(); // Java 14 is coming and we won't need a cast, so exciting.
                    ArrayData ac= new ArrayData();
                    ac.typeCode= ad.typeCode;
                    ac.dims= new int[ad.dims.length+1];
                    ac.dims[0]= nj;
                    System.arraycopy( ad.dims, 0, ac.dims, 1, ad.dims.length );
                    ac.array= Array.newInstance( ad.array.getClass(), nj );
                    Array.set( ac.array, j, ad.array );
                    o= ac;
                } else if ( e.getValue() instanceof Map ) {
                    Map accumulator1= new LinkedHashMap();
                    accumulate( accumulator1, (Map)e.getValue(), j, nj );
                    o= accumulator1;
                } else  {
                    Object d= e.getValue();
                    ArrayData ac= new ArrayData();
                    ac.dims= new int[] { nj };
                    Class t= getPrimativeClass( d.getClass() );
                    ac.array= Array.newInstance( t, nj );
                    Array.set( ac.array, j, d );
                    o= ac;
                }
                accumulator.put( e.getKey(), o );
            }
        }
        for ( Entry e: accumulator.entrySet() ) {
            Object o= rec.get(e.getKey());
            if ( o instanceof ArrayData ) {
                ArrayData ad= (ArrayData)o;
                ArrayData ac= (ArrayData)e.getValue();
                Array.set( ac.array, j, ad.array );
            } else if ( e.getValue() instanceof Map ) {
                accumulate( (Map)e.getValue(), (Map)o, j, nj);
            } else {
                ArrayData ac= (ArrayData)e.getValue();
                Array.set( ac.array, j, o );
            }
        }
    }
    
    private static class TypeDescStructure extends TypeDesc {
        ArrayDesc structArrayDesc;
        StructDesc structDesc;
        int offsetToData;
        boolean isSubstructure; // structure within a structure.
        /**
         * length of the data within the IDLSav file.
         */
        int _lengthBytes;
        
        @Override
        Object readData(ByteBuffer data) {
            LinkedHashMap result;
            int nj= structArrayDesc.nelements;
            if ( nj>1 ) {
                result= new LinkedHashMap<>();
                int iptr= offsetToData + ( isSubstructure ? 0 : 4 );
                int iptr0= iptr;
                for ( int j=0; j();
                int iptr= offsetToData + ( isSubstructure ? 0 : 4 );
                int iptr0= iptr;
                int iarray= 0;
                int istructure= 0;
                for ( int i=0; i arrayMap= new HashMap<>();
        Map structMap= new HashMap<>();
        
        int narray= 0;
        int nstruct= 0;
        for ( int i=0; i15 ) {
            throw new IllegalArgumentException("expected 0-14 for type code in readTypeDesc");
        }
        if ( ( varFlags & VARFLAG_STRUCT ) == VARFLAG_STRUCT ) {
            return readTypeDescStructure(typeDescBuf);
        } else if ( ( varFlags & VARFLAG_ARRAY ) == VARFLAG_ARRAY ) {
            return readTypeDescArray(typeDescBuf);
        } else {
            return readTypeDescScalar(typeDescBuf);
        }
    }
    
    /**
     * read the scalar, array, or structure at this position.  An
     * array is returned flattened, and readTypeDesc should be used
     * to unflatten it.  Structures are returned as a LinkedHashMap.
     * @param rec the byte buffer
     * @param offset offset into rec
     * @param vars map containing read data.
     * @return the read data.
     */
    private Object variable( ByteBuffer rec, int offset, Map vars) {
        logger.log( Level.FINER, "variable @ {0}", bufferOffsets.get(getKeyFor(rec)) );
        int type= rec.getInt(0+offset);
        if ( type!=RECTYPE_VARIABLE ) {
            throw new IllegalArgumentException("not a variable");
        }
        //printBuffer(rec);
        StringData varName= readString( rec, 20+offset );
        logger.log(Level.FINE, "variable name is {0}", varName );

        int nextField= 20 + varName._lengthBytes + offset;

        ByteBuffer data= slice(rec, nextField, rec.limit(), "typeDesc", "" );
        TypeDesc typeDesc= readTypeDesc( data );
        
        logger.log(Level.CONFIG, "variable_972 {0} {1,number,#} {2,number,#} {3}", new Object[]{data.position(), 0, data.limit(), varName});
        Object result= typeDesc.readData( data );
        
        vars.put( varName.string, result );
        
        return result;
        
    }
    
    /**
     * read the scalar, array, or structure at this position.  An
     * array is returned flattened, and readTypeDesc should be used
     * to unflatten it.  Structures are returned as a LinkedHashMap.
     * @param rec the byte buffer
     * @param offset offset into rec
     * @param vars map containing read data.
     * @return the read data.
     */
    private Object variable( FileChannel inch, int offset, Map vars) throws IOException {
        
        ByteBuffer rec= ByteBuffer.allocateDirect(512);  // length of variable name
        inch.read(rec,offset);
        
        logger.log( Level.FINER, "variable @ {0}", bufferOffsets.get(getKeyFor(rec)) );
        int type= rec.getInt(0+offset);
        if ( type!=RECTYPE_VARIABLE ) {
            throw new IllegalArgumentException("not a variable");
        }
        
        StringData varName= readString( rec, 20+offset );
        logger.log(Level.FINE, "variable name is {0}", varName );

        int nextField= 20 + varName._lengthBytes + offset;

        rec=null;
        ByteBuffer data= ByteBuffer.allocateDirect(nextField-offset);
        inch.read( rec, offset );
                //inch.//slice(rec, nextField, rec.limit(), "typeDesc", "" );
        TypeDesc typeDesc= readTypeDesc( data );
        
        logger.log(Level.CONFIG, "variable_972 {0} {1,number,#} {2,number,#} {3}", new Object[]{data.position(), 0, data.limit(), varName});
        Object result= typeDesc.readData( data );
        
        vars.put( varName.string, result );
        
        return result;
        
    }    
    
    /**
     * TODO: document me
     */
    private static final Map bufferOffsets= new HashMap<>();
    
    /**
     * Labels for each section of the 
     */
    private static final Map bufferLabels= new HashMap<>();
    
    private String nameFor( ByteBuffer buf ) {
        return bufferLabels.get(getKeyFor(buf));
    }
    
    private static Long getKeyFor( ByteBuffer buf ) {
        return ((long)buf.limit())*Integer.MAX_VALUE + buf.position();
    }
    
    private static Long getKeyFor( int position, int limit ) {
        return ((long)limit)* Integer.MAX_VALUE +position;
    }
    
    /**
     * slice out just the object 
     * @param src
     * @param position
     * @param limit
     * @param label
     * @return 
     */
    private ByteBuffer slice( ByteBuffer src, int position, int limit, String type, String label ) {
        if ( label==null ) throw new IllegalArgumentException("no label");
        Integer offset= bufferOffsets.get(getKeyFor(src));
        if ( offset!=null ) {
            logger.log(Level.CONFIG, "slice {0} {1,number,#} {2,number,#} {3}", 
                    new Object[]{ type, position+offset, limit+offset, label });
        } else {
            logger.log(Level.CONFIG, "slice {0} {1,number,#} {2,number,#} {3}", new Object[]{ type, position, limit, label });
            offset=0;
            if ( bufferLabels.get(getKeyFor(src))==null ) {
                bufferLabels.put(getKeyFor(src),"file");
            }
        }
        int position0= src.position();
        int limit0= src.limit();
        src.position(position);
        src.limit(limit);
        ByteBuffer r1= ByteBuffer.allocate(limit-position);
        r1.put(src.slice());
        r1.flip();
        src.limit(limit0);
        src.position(position0);
        
        bufferOffsets.put( getKeyFor(r1), position+offset );
        bufferLabels.put( getKeyFor(r1), label );
        return r1;
    }
    
    private ByteBuffer sliceLabel( ByteBuffer slice, int position, int limit, String type, String label ) {
        Integer offset= bufferOffsets.get(getKeyFor(position,limit));
        if ( offset!=null ) {
            logger.log(Level.CONFIG, "slice {0} {1,number,#} {2,number,#} {3}", 
                    new Object[]{ type, position+offset, limit+offset, label });
        } else {
            logger.log(Level.CONFIG, "slice {0} {1,number,#} {2,number,#} {3}", new Object[]{ type, position, limit, label });
            offset=0;
            if ( bufferLabels.get(getKeyFor(slice))==null ) {
                bufferLabels.put(getKeyFor(slice),"file");
            }
        }
        Long k= getKeyFor(position,position+slice.limit());
        bufferOffsets.put( k, position+offset );
        bufferLabels.put( k, label );
        return slice;
    }
    
    private String labelType( int type ) {
        switch (type) {
            case RECTYPE_TIMESTAMP:
                return "timeStamp";                
            case RECTYPE_VERSION:
                return "version";                
            case RECTYPE_VARIABLE:
                return "variable";
            case RECTYPE_ENDMARKER:
                return "endmarker";
            default:
                return "";
        }
    }
    
    public static ByteBuffer readFileIntoByteBuffer( File f ) throws IOException {
        RandomAccessFile aFile = new RandomAccessFile(f,"r");
        FileChannel inChannel = aFile.getChannel();
        long fileSize = inChannel.size();
        if ( fileSize>Integer.MAX_VALUE ) {
            throw new IllegalArgumentException("file is too large to read, and must be less than 2GB: "+f);
        }
        ByteBuffer buffer = ByteBuffer.allocate((int) fileSize);        
        int bytesRead= 0;
        while ( bytesRead readVars( ByteBuffer in ) throws IOException {
        
        //  2  ch.write(getBytesStr("SR"));

        //  1 ch.write(getBytesByte((byte) 0));
        //  1 ch.write(getBytesByte((byte) 4));

        int magic= in.getInt(0);
        if ( magic!=1397882884 ) {
            logger.warning("magic number is incorrect");
        }
        int pos= 4;
        
        Map result= new LinkedHashMap<>();
        
        ByteBuffer rec= readRecord( in, pos );
        while ( rec!=null ) {
            int type= rec.getInt(0);
            int nextPos= rec.getInt(4);
            if ( rec.getInt(8)!=0 ) {
                throw new IllegalArgumentException("records bigger than 2**32 bytes are not supported.");
            }
            logger.log(Level.CONFIG, "RecType: {0} Length: {1,number,#}", new Object[]{labelType(type), nextPos-pos});
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    logger.config("variable");
                    variable(rec, 0, result);
                    break;
                case RECTYPE_VERSION:
                    logger.config("version");
                    break;
                case RECTYPE_TIMESTAMP:
                    logger.config("timestamp");
                    break;
                default:
                    logger.config("???");
                    break;
            }
            pos= nextPos;
            rec= readRecord( in, pos );
        }
        return result;
    }

    public Map readVars( FileChannel inChannel ) throws IOException {
        
        //  2  ch.write(getBytesStr("SR"));

        //  1 ch.write(getBytesByte((byte) 0));
        //  1 ch.write(getBytesByte((byte) 4));

        checkMagic(inChannel);

        int pos= 4;
        
        Map result= new LinkedHashMap<>();
        
        ByteBuffer rec= readRecord( inChannel, pos );
        while ( rec!=null ) {
            int type= rec.getInt(0);
            int nextPos= rec.getInt(4);
            if ( rec.getInt(8)!=0 ) {
                throw new IllegalArgumentException("records bigger than 2**32 bytes are not supported.");
            }
            logger.log(Level.CONFIG, "RecType: {0} Length: {1,number,#}", new Object[]{labelType(type), nextPos-pos});
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    logger.config("variable");
                    variable(rec, 0, result);
                    break;
                case RECTYPE_VERSION:
                    logger.config("version");
                    break;
                case RECTYPE_TIMESTAMP:
                    logger.config("timestamp");
                    break;
                default:
                    logger.config("???");
                    break;
            }
            pos= nextPos;
            rec= readRecord( inChannel, pos );
        }
        return result;
    }
    
    /**
     * list the names in the IDLSav file.  This is only the supported
     * variable types.
     * @param in
     * @return the names found.
     * @throws IOException 
     */
    public String[] readVarNames( ByteBuffer in ) throws IOException {
        int magic= in.getInt(0);
        if ( magic!=1397882884 ) {
            logger.warning("magic number is incorrect");
        }
        int pos= 4;
        
        List names= new ArrayList<>();
        
        ByteBuffer rec= readRecord( in, pos );  
        while ( rec!=null ) {
            int type= rec.getInt(0);
            int nextPos= rec.getInt(4);
            logger.log(Level.CONFIG, "RecType: {0} Length: {1,number,#}", new Object[]{labelType(type), nextPos-pos});
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    logger.config("variable");
                    StringData varName= readString( rec, 20 );

                    int nextField= varName._lengthBytes;

                    ByteBuffer var= slice(rec, 20+nextField, rec.limit(), "var_x", "" );
                    
                    names.add(varName.string); 

                    break;
                case RECTYPE_VERSION:
                    logger.config("version");
                    break;
                case RECTYPE_TIMESTAMP:
                    logger.config("timestamp");
                    break;
                default:
                    logger.config("???");
                    break;
            }
            pos= nextPos;
            rec= readRecord( in, pos );
        }
        return names.toArray( new String[names.size()] );
    }
    
    /**
     * list the names in the IDLSav file.  This is only the supported
     * variable types.
     * @param in
     * @return the names found.
     * @throws IOException 
     */
    public String[] readVarNames( FileChannel inChannel ) throws IOException {
        checkMagic(inChannel);
        
        int pos= 4;
        
        List names= new ArrayList<>();
        
        ByteBuffer rec= readRecord( inChannel, pos );  
        while ( rec!=null ) {
            int type= rec.getInt(0);
            int nextPos= rec.getInt(4);
            logger.log(Level.CONFIG, "RecType: {0} Length: {1,number,#}", new Object[]{labelType(type), nextPos-pos});
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    logger.config("variable");
                    StringData varName= readString( rec, 20 );

                    int nextField= varName._lengthBytes;

                    ByteBuffer var= slice(rec, 20+nextField, rec.limit(), "var_x", "" );
                    
                    names.add(varName.string); 

                    break;
                case RECTYPE_VERSION:
                    logger.config("version");
                    break;
                case RECTYPE_TIMESTAMP:
                    logger.config("timestamp");
                    break;
                default:
                    logger.config("???");
                    break;
            }
            pos= nextPos;
            rec= readRecord( inChannel, pos );
        }
        return names.toArray( new String[names.size()] );
    }    
    
    /**
     * scan through the IDLSav and return just the one variable.
     * @param in the IDLSav mapped into a NIO ByteBuffer.
     * @param name the variable name to look for.
     * @return
     * @throws IOException 
     */
    public Object readVar( ByteBuffer in, String name ) throws IOException {
        int magic= in.getInt(0);
        if ( magic!=1397882884 ) {
            logger.warning("magic number is incorrect, file should start with should be 1397882884");
        }
        if ( in.order()!=ByteOrder.BIG_ENDIAN ) {
            throw new IllegalArgumentException("buffer must be big endian");
        }
        if ( in.position()==0 ) {
            logger.log(Level.CONFIG, "readVar {0} buffer size: {1,number,#}", new Object[] { name, in.limit() } );
        }

        bufferOffsets.put( getKeyFor(in), 0 );
        bufferLabels.put( getKeyFor(in), "" );

        int pos= 4;
        String name0= name; // keep name for reference.
        ByteBuffer rec= readRecord( in, pos );
        
        while ( rec!=null ) {
    
            int offset = bufferOffsets.get(getKeyFor(rec));
        
            int type= rec.getInt(0);
            int nextPos= rec.getInt(4);
            logger.log(Level.CONFIG, "RecType: {0} Length: {1,number,#}", new Object[]{labelType(type), nextPos-pos});
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    StringData varName= readString( rec, 20 );
                    logger.log(Level.CONFIG, "variable {0} {1,number,#} {2,number,#} {3}", 
                            new Object[] { type, pos, nextPos, varName } );
                    String rest= null;
                    int i= name.indexOf(".");
                    if ( i>-1 ) {
                        rest= name.substring(i+1);
                        name= name.substring(0,i);
                    }
                    if ( i==-1 ) {
                        if ( varName.string.equals(name) ) {
                            Map result= new HashMap<>();
                            variable( in, offset, result);
                            return result.get(name);
                        }
                    } else {
                        if ( varName.string.equals(name) ) {
                            Map result= new HashMap<>();
                            variable( in, offset, result );
                            Map res= (Map) result.get(name);
                            assert rest!=null;
                            i= rest.indexOf('.');
                            while ( i>-1 ) {
                                res= (Map)res.get( rest.substring(0,i) );
                                rest= rest.substring(i+1);
                                i= rest.indexOf('.');
                            }
                            return res.get(rest);
                        }
                    }
                    break;
                case RECTYPE_VERSION:
                    logger.config("version");
                    break;
                case RECTYPE_TIMESTAMP:
                    logger.config("timestamp");
                    break;
                case RECTYPE_PROMOTE64:
                    logger.config("promote64");
                    throw new IllegalArgumentException("promote64 is not supported.");
                default:
                    logger.config("???");
                    break;
            }
            pos= nextPos;
            rec= readRecord( in, pos );
            
            
        }
        return null;        
        
    }  
    
    /**
     * scan through the IDLSav and return just the one variable.
     * @param inch FileChannel for the IDLSav.
     * @param name the variable name to look for.
     * @return
     * @throws IOException 
     */
    public Object readVar( FileChannel inch, String name ) throws IOException {
        checkMagic(inch);

        bufferOffsets.put( getKeyFor(0,0), 0 );
        bufferLabels.put( getKeyFor(0,0), "" );

        int pos= 4;
        String name0= name; // keep name for reference.
        ByteBuffer rec= readRecord( inch, pos );
        
        while ( rec!=null ) {
    
            int offset = bufferOffsets.get(getKeyFor(rec));
        
            int type= rec.getInt(0);
            int nextPos= rec.getInt(4);
            logger.log(Level.CONFIG, "RecType: {0} Length: {1,number,#}", new Object[]{labelType(type), nextPos-pos});
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    StringData varName= readString( rec, 20 );
                    logger.log(Level.CONFIG, "variable {0} {1,number,#} {2,number,#} {3}", 
                            new Object[] { type, pos, nextPos, varName } );
                    String rest= null;
                    int i= name.indexOf(".");
                    if ( i>-1 ) {
                        rest= name.substring(i+1);
                        name= name.substring(0,i);
                    }
                    if ( i==-1 ) {
                        if ( varName.string.equals(name) ) {
                            Map result= new HashMap<>();
                            variable( inch, offset, result);
                            return result.get(name);
                        }
                    } else {
                        if ( varName.string.equals(name) ) {
                            Map result= new HashMap<>();
                            variable( inch, offset, result );
                            Map res= (Map) result.get(name);
                            assert rest!=null;
                            i= rest.indexOf('.');
                            while ( i>-1 ) {
                                res= (Map)res.get( rest.substring(0,i) );
                                rest= rest.substring(i+1);
                                i= rest.indexOf('.');
                            }
                            return res.get(rest);
                        }
                    }
                    break;
                case RECTYPE_VERSION:
                    logger.config("version");
                    break;
                case RECTYPE_TIMESTAMP:
                    logger.config("timestamp");
                    break;
                case RECTYPE_PROMOTE64:
                    logger.config("promote64");
                    throw new IllegalArgumentException("promote64 is not supported.");
                default:
                    logger.config("???");
                    break;
            }
            pos= nextPos;
            rec= readRecord( inch, pos );
            
        }
        return null;        
        
    }  
        
    
    /**
     * scan through the IDLSav and retrieve information about the array.
     * @param in the idlsav loaded into a ByteBuffer.
     * @param name the name of the array
     * @return
     * @throws IOException 
     */
    public TagDesc readTagDesc( ByteBuffer in, String name ) throws IOException {
        int magic= in.getInt(0);
        if ( magic!=1397882884 ) {
            logger.warning("magic number is incorrect");
        }
        int pos= 4;
        
        ByteBuffer rec= readRecord( in, pos );
        while ( rec!=null ) {
            int type= rec.getInt(0);
            int nextPos= rec.getInt(4);
            logger.log(Level.CONFIG, "RecType: {0} Length: {1,number,#}", new Object[]{labelType(type), nextPos-pos});
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    logger.config("variable");
                    StringData varName= readString( rec, 20 );
                    if ( name.startsWith(varName.string+".") || name.equals(varName.string) ) {
                        int nextField= varName._lengthBytes;
                        ByteBuffer var= slice(rec, 20+nextField, rec.limit(), "variable", varName.string );
                        if ( var.getInt(0)==8 ) { // TODO: what is 8?
                            if ( ( var.getInt(4) & VARFLAG_STRUCT ) == VARFLAG_STRUCT ) {
                                TypeDescStructure typeDescStructure= readTypeDescStructure(var);
                                if ( name.equals(varName.string) ) {
                                    return typeDescStructure.structDesc;
                                } else {
                                    return findStructureTag( typeDescStructure.structDesc, name.substring(varName.string.length()+1) );
                                }
                            } else {
                                return readTypeDescArray(var).arrayDesc;
                            }
                        } else {
                            if ( ( var.getInt(4) & VARFLAG_ARRAY ) == VARFLAG_ARRAY ) {
                                TagDesc dd= readTypeDescArray(var).arrayDesc;
                                dd.typecode= readTypeDescArray(var).typeCode;
                                return dd;
                            } else {
                                return readTagDesc(var);
                            }
                        }
                    }
                    break;
                case RECTYPE_VERSION:
                    logger.config("version");
                    break;
                case RECTYPE_TIMESTAMP:
                    logger.config("timestamp");
                    break;
                default:
                    logger.config("???");
                    break;
            }
            pos= nextPos;
            rec= readRecord( in, pos );
        }
        return null;        
    }

    public static boolean checkMagic( FileChannel inChannel ) throws IOException {
        ByteBuffer buf= ByteBuffer.allocate(4);
        if ( !(inChannel.read(buf)==4) ) {
            throw new IllegalArgumentException("not 4 bytes");
        }
        int magic= buf.getInt(0);
        if ( magic!=1397882884 ) {
            logger.warning("magic number is incorrect");
            return false;
        } else {
            return true;
        }
    }
    
    /**
     * scan through the IDLSav and retrieve information about the array.
     * @param inch the FileChannel for the idlsav
     * @param name the name of the array
     * @return
     * @throws IOException 
     */    
    /**
     * scan through the IDLSav and retrieve information about the array.
     * @param inch the FileChannel for the idlsav
     * @param name the name of the array
     * @return
     * @throws IOException 
     */
    public TagDesc readTagDesc( FileChannel inch, String name ) throws IOException {
        checkMagic(inch);
        
        int pos= 4;
        
        ByteBuffer rec= readRecord( inch, pos );
        while ( rec!=null ) {
            int type= rec.getInt(0);
            int nextPos= rec.getInt(4);
            logger.log(Level.CONFIG, "RecType: {0} Length: {1,number,#}", new Object[]{labelType(type), nextPos-pos});
            switch ( type ) {
                case RECTYPE_VARIABLE:
                    logger.config("variable");
                    StringData varName= readString( rec, 20 );
                    if ( name.startsWith(varName.string+".") || name.equals(varName.string) ) {
                        int nextField= varName._lengthBytes;
                        ByteBuffer var= slice(rec, 20+nextField, rec.limit(), "variable", varName.string );
                        if ( var.getInt(0)==8 ) { // TODO: what is 8?
                            if ( ( var.getInt(4) & VARFLAG_STRUCT ) == VARFLAG_STRUCT ) {
                                TypeDescStructure typeDescStructure= readTypeDescStructure(var);
                                if ( name.equals(varName.string) ) {
                                    return typeDescStructure.structDesc;
                                } else {
                                    return findStructureTag( typeDescStructure.structDesc, name.substring(varName.string.length()+1) );
                                }
                            } else {
                                return readTypeDescArray(var).arrayDesc;
                            }
                        } else {
                            if ( ( var.getInt(4) & VARFLAG_ARRAY ) == VARFLAG_ARRAY ) {
                                TagDesc dd= readTypeDescArray(var).arrayDesc;
                                dd.typecode= readTypeDescArray(var).typeCode;
                                return dd;
                            } else {
                                return readTagDesc(var);
                            }
                        }
                    }
                    break;
                case RECTYPE_VERSION:
                    logger.config("version");
                    break;
                case RECTYPE_TIMESTAMP:
                    logger.config("timestamp");
                    break;
                default:
                    logger.config("???");
                    break;
            }
            pos= nextPos;
            rec= readRecord( inch, pos );
        }
        return null;        
    }
    
    private static void arrayToString( Object o, StringBuilder b ) {
        char delim=',';
        for ( int j=0; j<4; j++ ) {
            Object i= Array.get(o,j);
            if ( i.getClass().isArray() ) {
                delim=';';
                if ( j>0 ) b.append(delim);
                arrayToString( i, b );
            } else {
                if ( j>0 ) b.append(delim);
                b.append(i.toString());
            }
        }
        if ( Array.getLength(o)>4 ) {
            b.append(delim);
            b.append("...");
        }
    }
    
//    public static void main( String[] args ) throws IOException {
//        Logger logger= Logger.getLogger("autoplot.idlsav");
//        //logger.setLevel( Level.FINE );
//        Handler h= new ConsoleHandler();
//        h.setLevel(Level.ALL);
//        logger.addHandler(h);
//            
////        FileOutputStream fos = new FileOutputStream(new File("/tmp/test.autoplot.idlsav"));
////        
////        WriteIDLSav widls= new WriteIDLSav();
////        //widls.addVariable( "wxyz", new double[] { 120,100,120,45,46,47,48,49,120,100,120 } );
////        widls.addVariable( "abcd", 240 );
////        //widls.addVariable( "oneval", 19.95 );
////        widls.write(fos);
////        
////        fos.close();
//
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/simple.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/vnames.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/scalars.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/arrayVsScalar.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/floats.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                /home/jbf/public_html/autoplot/data/sav/structureOfLonarr.idlsav    "/home/jbf/public_html/autoplot/data/sav/doublearray.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                  "/home/jbf/public_html/autoplot/data/sav/structureOfLonarr.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/arrayOfStruct.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/arrayOfStruct1Var.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/structure.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/structureWithinStructure.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/stuctOfStruct.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/stuctOfStructOfStruct.idlsav","r");
//        //RandomAccessFile aFile = new RandomAccessFile(
//        //                    "/home/jbf/public_html/autoplot/data/sav/stuctOfStructOfStruct.idlsav","r");
//        RandomAccessFile aFile = new RandomAccessFile(
//                            "/home/jbf/ct/autoplot/data/sav/kristoff/test_fit.idlsav","r");
//                
//        FileChannel inChannel = aFile.getChannel();
//        long fileSize = inChannel.size();
//        
//        ByteBuffer buffer = ByteBuffer.allocate((int) fileSize);
//        int bytesRead= 0;
//        while ( bytesRead vars= new ReadIDLSav().readVars(buffer);
//        
//        for ( Entry v : vars.entrySet() ) {
//            System.err.println( v );
//            if ( v.getValue() instanceof Map ) {
//                Map m= (Map)v.getValue();
//                for ( Entry j : m.entrySet() ) {
//                    Object k= j.getValue();
//                    if ( k instanceof ArrayData ) {
//                        System.err.print(j.getKey()+":");
//                        StringBuilder b= new StringBuilder();
//                        arrayToString( ((ArrayData)k).array, b);
//                        System.err.println(b.toString());
//                    } else if ( k==null ) {
//                        System.err.println("<>");
//                    } else {
//                        System.err.println(k.toString());
//                    }
//                }
//            } else {
//                System.err.println(v.getValue());
//            }
//        }
//        
//    }

}