/* This Java package, org.autoplot.das2Stream is part of the Autoplot application
*
* Autoplot is free software; you can redistribute it and/or modify it under the
* terms of the GNU General Public License version 2 as published by the Free
* Software Foundation, with the Classpath exception below.
*
* This program is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License version 2
* containing the Classpath exception clause along with this library; if not, write
* to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
* 02111-1307 USA
*
* Classpath Exception
* -------------------
* The copyright holders designate this particular java package as subject to the
* "Classpath" exception as provided here.
*
* Linking this package statically or dynamically with other modules is making a
* combined work based on this package. Thus, the terms and conditions of the GNU
* General Public License cover the whole combination.
*
* As a special exception, the copyright holders of this package give you
* permission to link this package with independent modules to produce an
* application, regardless of the license terms of these independent modules, and
* to copy and distribute the resulting application under terms of your choice,
* provided that you also meet, for each independent module, the terms and
* conditions of the license of that module. An independent module is a module
* which is not derived from or based on this package. If you modify this package,
* you may extend this exception to your version of the package, but you are not
* obligated to do so. If you do not wish to do so, delete this exception
* statement from your version.
*/
package org.das2.qstream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.das2.datum.LoggerManager;
import org.das2.datum.TimeLocationUnits;
import org.das2.datum.Units;
import org.das2.datum.UnitsUtil;
import org.das2.qds.DataSetOps;
import org.das2.qds.MutablePropertyDataSet;
import org.das2.qds.QDataSet;
import org.das2.qds.SemanticOps;
import org.das2.qds.ops.Ops;
import org.w3c.dom.DOMImplementation;
import org.w3c.dom.Document;
import org.w3c.dom.ls.DOMImplementationLS;
import org.w3c.dom.ls.LSSerializer;
/** Base class for QDataSet to das2 stream serializers
*
*/
abstract public class QdsToD2sStream {
// Private ////////////////////////////////////////////////////////////////
private static final Logger log = LoggerManager.getLogger("qstream");
// Public interface //////////////////////////////////////////////////////
public static final String FORMAT_2_2 = "2.2";
public static final String FORMAT_2_3_BASIC = "2.3/basic";
public static final String FORMAT_2_4_GENERAL = "2.4/general";
public static final String[] formats = {FORMAT_2_2, FORMAT_2_3_BASIC};
public static final int DEFAUT_FRAC_SEC = 3;
public static final int DEFAUT_SIG_DIGIT = 5;
public static final int FIXED_PKT_TAGS = 0; // original tag format
public static final int VAR_PKT_TAGS = 1; // new variable packet tags
/** Initialize a binary QDataSet to das2 stream exporter
*/
public QdsToD2sStream(){
bBinary = true;
}
/** Initialize a text QDataSet to das2 stream exporter
*
* @param genSigDigits The number of significant digits used for general
* text value data output. If you don't know what else to use, 5 is
* typically a fine precision without being ridiculous.
*
* @param fracSecDigits The number of fractional seconds digits to
* use for ISO-8601 date-time values. The number of fraction seconds
* can be set as low as 0 and as high as 12 (picoseconds). If
* successive time values vary by less than the specified precision
* (but not 0, repeats are accepted) then stream writing fails. Use 3 (i.e.
* microseconds) if you don't know what else to choose.
*/
public QdsToD2sStream(int genSigDigits, int fracSecDigits){
bBinary = false;
if(genSigDigits > 1){
if(genSigDigits > 16){
throw new IllegalArgumentException(String.format(
"Number of significant digits in the output must be between 2 "
+ "and 17, received %d", genSigDigits
));
}
nSigDigit = genSigDigits;
}
if(fracSecDigits >= 0){
if(fracSecDigits < 0 || fracSecDigits > 12){
throw new IllegalArgumentException(String.format(
"Number of fractional seconds digits in the output must be "
+ "between 0 and 12 inclusive, received %d", fracSecDigits
));
}
nSecDigit = fracSecDigits;
}
}
/** Determine if the given dataset can be written
*
* @param qds
* @return
*/
public abstract boolean canWrite(QDataSet qds);
/** Write the given dataset
*
* @param qds
* @param os
* @return
* @throws java.io.IOException
*/
public abstract boolean write(QDataSet qds, OutputStream os)
throws IOException;
// protected //////////////////////////////////////////////////////////
/** Write a UTF-8 header string onto the stream, if packet ID == 0 then
* write a stream header
*
* @param os The output stream
* @param nTagType Either FIXED_PKT_TAGS or VAR_PKT_TAGS
* @param nPktId
* @param sHdr - The header string to write for this packet id
* @throws UnsupportedEncodingException
* @throws IOException
*/
static void writeHeader(OutputStream os, int nTagType, int nPktId, String sHdr)
throws UnsupportedEncodingException, IOException{
byte[] aHdr = sHdr.getBytes(StandardCharsets.UTF_8);
String sTag;
if(nTagType == VAR_PKT_TAGS){
if(nPktId == 0)
sTag = String.format("|Hs||%d|", aHdr.length);
else
sTag = String.format("|Hx|%d|%d|", nPktId, aHdr.length);
}
else{
sTag = String.format("[%02d]%06d", nPktId, aHdr.length);
}
byte[] aTag = sTag.getBytes(StandardCharsets.US_ASCII);
os.write(aTag);
os.write(aHdr);
}
// Handling for text output, throw this back to the user if they want text
// output. The reason not to just do our best to figure it out on our own
// is that it changes the packet header definitions, which we don't want to
// do when writing cache files.
protected int nSigDigit = DEFAUT_SIG_DIGIT;
protected int nSecDigit = DEFAUT_FRAC_SEC;
protected boolean bBinary;
/** Determine and hold the information needed to transfer values out of a
* given QDataSet into a byte buffer. The byte order is always native
* endian
*/
static protected class QdsXferInfo {
QDataSet qds;
TransferType transtype;
/** Figure out how to represent the values. In general everything is
* output as a float unless it's an epoch time type.
*
* @param _qds The QDataset
* @param bBinary True if user wanted binary data
* @param nGenDigit Number of significant digits for general text values
* @param nFracSec Number of fractional seconds for time text values
*/
QdsXferInfo(
QDataSet _qds, boolean bBinary, int nGenSigDigit, int nFracSec
){
qds = _qds;
if(bBinary){ nGenSigDigit = -1; nFracSec = -1; }
Units units = (Units) qds.property(QDataSet.UNITS);
if((units != null) && (units instanceof TimeLocationUnits)){
if(nFracSec < 0)
transtype = new DoubleTransferType();
else
transtype = new TransferTimeAtPrecision(units, nFracSec);
}
else{
//Non epoch stuff
Class c = DataSetOps.getComponentType(qds);
if((c == int.class)||(c == long.class)){
if(bBinary){
if(c == int.class) transtype = new IntegerTransferType();
else transtype = new LongTransferType();
}
else{
// Yea, couldn't be something like QDataSet.max()
QDataSet dsExt = Ops.extent(qds);
double rMax = Math.abs(dsExt.value(0));
double rMin = Math.abs(dsExt.value(1));
if(rMin > rMax) rMax = rMin;
int nChars = (int)(Math.ceil( Math.log10( rMax) ) + 2);
transtype = new AsciiIntegerTransferType(nChars);
}
}
else{
if(nGenSigDigit < 0)
transtype = new FloatTransferType();
else
transtype = new TransferSciNotation(nGenSigDigit);
}
}
}
/** Get a size independent type name
* @return the type with any size information stripped off */
public String name(){
// Since float and double indicate size, use integer names
// that indicate size as well for consistency.
if(transtype.name().equals("int8")) return "long";
else return transtype.name().replaceAll("\\d","");
}
/** @return the size in bytes of each output value */
public int size(){ return transtype.sizeBytes(); }
/** Get the number items in a single X-axis slice of this dataset the
* X-axis is synonymous with the QDataSet 0th axis for now.
* @param i - The X point at which to get the items
* @return
*/
public int xSliceItems(int i) throws IOException{
int nItems = 0;
switch(qds.rank()){
case 1: return 1;
case 2: return qds.length(i);
case 3:
for(int j=0; j < qds.length(i); ++j)
nItems += qds.length(i,j);
return nItems;
case 4:
// Das2 can't really do this, but calc anyway
for(int j=0; j < qds.length(i); ++j){
for(int k=0; k < qds.length(i,j); ++k)
nItems += qds.length(i,j,k);
}
return nItems;
default:
throw new IOException(String.format(
"Can't stream rank %d data with this format.", qds.rank()
));
}
}
/** Get the number of bytes needed to hold an x-slice of a single
*
* @param i
* @return
*/
public int xSliceBytes(int i) throws IOException{
return transtype.sizeBytes() * xSliceItems(i);
}
}
// Hold the information for serializing a single packet and it's dependencies
static protected class PacketXferInfo {
Document doc; // Overall header document
List<QdsXferInfo> lDsXfer;
PacketXferInfo(Document _doc, List<QdsXferInfo> _lDsXfer ){
doc = _doc; lDsXfer = _lDsXfer;
}
int datasets(){ return lDsXfer.size(); }
// Calc length of buffer needed to hold a single slice in the 0th
// dimension across all qdatasets.
int xSliceBytes() throws IOException {
int nLen = 0;
for(QdsXferInfo qi: lDsXfer)
nLen += qi.xSliceBytes(0);
return nLen;
}
}
// Send Bundle data
protected void writeData(
OutputStream out, int nTagType, int iPktId, PacketXferInfo pktXfer
) throws IOException {
WritableByteChannel channel = Channels.newChannel(out);
// Make the Xslice record tag. For now these all have the same length
// if das2/general format is ever introduced will have to check the
// length for each X-slice.
String sRecTag;
if(nTagType == VAR_PKT_TAGS){
sRecTag = String.format("|Dx|%d|%d|", iPktId, pktXfer.xSliceBytes());
}
else{
sRecTag = String.format(":%02d:", iPktId);
}
byte[] aRecTag = sRecTag.getBytes(StandardCharsets.US_ASCII);
int nBufLen = pktXfer.xSliceBytes() + aRecTag.length;
byte[] aBuf = new byte[nBufLen];
ByteBuffer buffer = ByteBuffer.wrap(aBuf);
buffer.order(ByteOrder.nativeOrder()); // Since we have a choice, use native
buffer.put(aRecTag); // tag stays in buffer
int nPkts = pktXfer.lDsXfer.get(0).qds.length();
for(int iPkt = 0; iPkt < nPkts; ++iPkt){
QdsXferInfo qi = null;
for(int iDs = 0; iDs < pktXfer.datasets(); ++iDs){
qi = pktXfer.lDsXfer.get(iDs);
switch(qi.qds.rank()){
case 1:
qi.transtype.write(qi.qds.value(iPkt), buffer);
break;
case 2:
// Does not assume cubic, if stream writers expect cubic, they
// need to check for it before calling this function
for(int iVal = 0; iVal < qi.qds.length(iPkt); ++iVal)
qi.transtype.write(qi.qds.value(iPkt, iVal), buffer);
break;
case 3:
// Does not assume cubic, if stream writers expect cubic, they
// need to check for it before calling this function
for(int iVal = 0; iVal < qi.qds.length(iPkt); ++iVal)
for(int jVal = 0; jVal < qi.qds.length(iPkt, iVal); ++jVal)
qi.transtype.write(qi.qds.value(iPkt, iVal, jVal), buffer);
break;
default:
assert(false);
}
}
if(qi != null && qi.transtype.isAscii())
buffer.put(nBufLen - 1, (byte)'\n');
buffer.flip();
channel.write(buffer);
buffer.position(aRecTag.length);
}
}
// Secondary helpers functions below here //////////////////////////////////
protected String makeNameFromUnits(Units units)
{
if(units == null) return "";
if(UnitsUtil.isTimeLocation(units)) return "time";
if(units.isConvertibleTo(Units.meters)) return "length";
if(units.isConvertibleTo(Units.hertz)) return "frequnecy";
if(units.isConvertibleTo(Units.eV)) return "energy";
if(units.isConvertibleTo(Units.degrees)) return "angle";
if(units.isConvertibleTo(Units.seconds)) return "interval";
if(units.isConvertibleTo(Units.bytes)) return "size";
if(units.isConvertibleTo(Units.cm_2s_1keV_1)) return "flux";
if(units.isConvertibleTo(Units.kelvin)) return "temperature";
String sUnits = units.toString();
// I wish there was some sort of units canonicalization function
// Get frequency typo's
if(sUnits.equalsIgnoreCase("mhz")) return "frequency";
if(sUnits.equalsIgnoreCase("khz")) return "frequency";
if(sUnits.equalsIgnoreCase("hz")) return "frequency";
// So where is spectral density? Try some wierd stuff
if(sUnits.equalsIgnoreCase("V!a2!nm!a-2!nHz!a-1!n")) // yes, this happens
return "e_spec_dens";
if(sUnits.equalsIgnoreCase("nT!a2!nHz!a-1!n")) // yes, this happens too
return "b_spec_dens";
return "";
}
protected boolean _stripDotProps(QDataSet qds){
// Ephemeris data (x-multi-y) ported in from das1 add many redundant
// properties get rid of those if this is a bundle_1 dataset and each
// sub-item is rank 1
if(qds.property(QDataSet.BUNDLE_1) != null){
for(int i = 0; i < qds.length(0); ++i){
QDataSet ds = DataSetOps.slice1(qds, i);
if(ds.rank() != 1) return false;
}
}
else{
return false;
}
return true;
}
static String xmlDocToStr(Document doc)
{
DOMImplementation imp = doc.getImplementation();
DOMImplementationLS ls = (DOMImplementationLS)imp.getFeature("LS", "3.0");
LSSerializer serializer = ls.createLSSerializer();
//DOMStringList props = serializer.getDomConfig().getParameterNames();
serializer.getDomConfig().setParameter("format-pretty-print", true);
serializer.getDomConfig().setParameter("xml-declaration", false);
String sDoc = serializer.writeToString(doc);
return sDoc;
}
protected Document newXmlDoc() {
try {
DocumentBuilder bldr =
DocumentBuilderFactory.newInstance().newDocumentBuilder();
return bldr.newDocument();
} catch (ParserConfigurationException pce) {
throw new RuntimeException(pce);
}
}
protected static class Sequence1D {
String sMinval = null;
String sInterval = null;
}
/** Determine if a rank 1 dataset is really just a sequence
*
* @param qds
* @param rMaxJitter Recommend 1e-4 if no other values comes to mind
* @return A Sequence1D object with minval and interval set.
*/
protected Sequence1D getSequenceRank1(QDataSet qds, double rMaxJitter)
{
if(qds.rank() != 1) return null;
if(qds.length() < 2) return null;
double rMin = qds.value(0);
// Try to use sequence representation if we can, allow for jitter in
// intervals of one part in the max jitter
double rInterval = (qds.value( qds.length() - 1) - rMin)/(qds.length() - 1);
double rNextInterval, rAvg, rDelta, rJitter;
for(int i = 2; i < qds.length(); ++i){
rNextInterval = qds.value(i) - qds.value(i - 1);
if(rNextInterval != rInterval){
rAvg = Math.abs(rNextInterval + rInterval)/2;
rDelta = Math.abs(rNextInterval - rInterval);
if(rDelta == 0.0) continue;
if(rAvg == 0.0)
return null;
rJitter = rDelta/ rAvg;
if(rJitter > rMaxJitter)
return null;
}
}
// Get my format string
String sFmt = String.format("%%.%de", nSigDigit - 1);
Sequence1D seq = new Sequence1D();
seq.sMinval = String.format(sFmt, rMin);
seq.sInterval = String.format(sFmt, rInterval);
return seq;
}
///////////////////////////////////////////////////////////////////////////
// Das2 specific dataset properties and functions that could just be an add
// on for qdataset. These could be move somewhere else.
/** Determine the name of the das2 axis on which values from a dataset
* would typically be plotted.
*
* This is a duck-typing check. Which looks at the number of dependencies
* and planes in a dataset. If the dataset has a PLANE_0 property, the axis
* of the PLANE_0 values is returned instead of the axis of the primary
* dataset.
*
* @param qds
* @return one of "x", "y", "z","w" or null if we can't figure it out.
*/
public static String getQdsAxis(QDataSet qds){
//If join just look at the first element of the join
if(SemanticOps.isJoin(qds)) qds = DataSetOps.slice0(qds, 0);
//If see if the bundle is bigger than size 1.
if(SemanticOps.isBundle(qds)){
if(qds.rank() == 1){ //trival bundle
qds = DataSetOps.slice0(qds, 0);
// now handle as single dataset below
}
else{
// I'm continuing the false assmption that paramenter space
// dimensionality ~= rank which is the the core of the CDF/QDataSet
// path dataset problem. We will have to face this head on someday.
// The only indicator that a rank 1 item is actually a Z value is the
// existance of a PLANE_0 property.
int nMaxRank = 0;
for(int i = 0; i < qds.length(); ++i){
QDataSet ds = DataSetOps.slice0(qds, i);
int nRank = ds.rank();
if(ds.property(QDataSet.PLANE_0) != null)
//throw new UnsupportedOperationException(
// "Cannot determine the canonical axis for bundles which contain "+
// "datasets which have a PLANE_0 property."
//);
return null;
if(nRank > nMaxRank) nMaxRank = ds.rank();
}
if(nMaxRank == 1) return "y";
if(nMaxRank == 2) return "z";
return null;
}
}
//Okay, not a bundle. To decide our axis just check our dependencies
//rank, and planes. Here's the swiss cheese check:
// rank 0 -> X
// rank 1 with no depend_0 -> X
// rank 1 with a depend_0 -> Y
// rank 1 with a depend_0 and plane_0 -> Z (rem to pull out plane for Z's)
// rank 2 -> Z
switch(qds.rank()){
case 0: return "x";
case 2: return "z";
case 3: return "w";
case 1:
// TODO: Read input to see how x/y/z data are handled
if(qds.property(QDataSet.DEPEND_0) == null) return "x";
if(qds.property(QDataSet.PLANE_0) == null) return "y";
return "z";
}
return null;
}
private static final List<String> lSimpleKeys;
static {
lSimpleKeys = new ArrayList<>();
lSimpleKeys.add(QDataSet.NAME);
lSimpleKeys.add(QDataSet.UNITS);
lSimpleKeys.add(QDataSet.FORMAT);
lSimpleKeys.add(QDataSet.SCALE_TYPE);
lSimpleKeys.add(QDataSet.LABEL);
lSimpleKeys.add(QDataSet.DESCRIPTION);
lSimpleKeys.add(QDataSet.FILL_VALUE);
lSimpleKeys.add(QDataSet.VALID_MIN);
lSimpleKeys.add(QDataSet.VALID_MAX);
lSimpleKeys.add(QDataSet.TYPICAL_MIN);
lSimpleKeys.add(QDataSet.TYPICAL_MAX);
lSimpleKeys.add(QDataSet.USER_PROPERTIES);
}
/** Only copy over the simple properties of a dataset, ignore the structural
* items such as depend, offset, axis, reference, bundle etc.
*
* @param dsDest
* @param dsSrc
* @return
*/
protected int copySimpleProps(MutablePropertyDataSet dsDest, QDataSet dsSrc)
{
int nAdded = 0;
Object oVal;
for(String sKey: lSimpleKeys){
if( (oVal = dsSrc.property(sKey)) != null){
dsDest.putProperty(sKey, oVal);
nAdded += 1;
}
}
return nAdded;
}
}