package org.das2.qstream;
import java.nio.ByteBuffer;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import javax.xml.xpath.XPathFactoryConfigurationException;
import org.das2.datum.Datum;
import org.das2.datum.EnumerationUnits;
import org.das2.qds.ArrayDataSet;
import org.das2.qds.BundleDataSet;
import org.das2.qds.DDataSet;
import org.das2.qds.DataSetUtil;
import org.das2.qds.JoinDataSet;
import org.das2.qds.MutablePropertyDataSet;
import org.das2.qds.QDataSet;
import org.das2.qds.SemanticOps;
import org.das2.qds.ops.Ops;
import org.das2.qds.util.DataSetBuilder;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
/**
* reads a stream and produces QDataSets representing the data found on the stream. The stream is read in, and
* then getDataSet or getDataSet(name) is called to retrieve datasets.
*
* @author jbf
*/
public class QDataSetStreamHandler implements StreamHandler {
private static final Logger logger = Logger.getLogger("qstream");
public static final String BUILDER_JOIN_CHILDREN = "join";
Map<String, DataSetBuilder> builders;
Map<String, JoinDataSet> joinDataSets;
Map<String, String[]> bundleDataSets;
Map<String, Integer> ranks;
XPathFactory factory = getXPathFactory();
XPath xpath = factory.newXPath();
String dsname;
boolean readPackets = true;
// each enumeration units need to be parsed within the context of the stream, not the entire session.
EnumerationUnitsSerializeDelegate enumerationUnitsSerializeDelegate= new EnumerationUnitsSerializeDelegate();
public QDataSetStreamHandler() {
builders = new HashMap<>();
joinDataSets = new HashMap<>();
bundleDataSets = new HashMap<>();
ranks= new HashMap<>();
}
/**
* Matlab uses net.sf.saxon.xpath.XPathEvaluator by default, so we explicitly look for the Java 6 one.
*
* @return com.sun.org.apache.xpath.internal.jaxp.XPathFactoryImpl, probably.
*
* This is a copy of DataSourceUtil.getXPathFactory.
*/
private static XPathFactory getXPathFactory() {
XPathFactory xpf;
try {
xpf = XPathFactory.newInstance(XPathFactory.DEFAULT_OBJECT_MODEL_URI, "com.sun.org.apache.xpath.internal.jaxp.XPathFactoryImpl", null);
} catch (XPathFactoryConfigurationException ex) {
xpf = XPathFactory.newInstance();
logger.log(Level.INFO, "using default xpath implementation: {0}", xpf.getClass());
}
return xpf;
}
/**
* return a list of available datasets
*
* @return
*/
public List<String> getDataSetNames() {
return new ArrayList<>(builders.keySet());
}
/**
* return a list of available datasets and their label (or name if not available).
*
* @return
*/
public Map<String, String> getDataSetNamesAndDescriptions() {
Map<String, String> result = new LinkedHashMap();
for (Entry<String, DataSetBuilder> e : builders.entrySet()) {
DataSetBuilder b = e.getValue();
String name = null;
String n;
n = (String) b.getProperties().get(QDataSet.NAME);
if (n != null) {
name = n;
}
n = (String) b.getProperties().get(QDataSet.LABEL);
if (n != null) {
name = n;
}
result.put(e.getKey(), name);
}
return result;
}
/**
* create a builder, figuring out if there's an implicit streaming dimension.
*
* @param rank
* @param qube
* @return
*/
DataSetBuilder createBuilder(int rank, int[] qube) {
DataSetBuilder result;
if (rank == qube.length) {
switch (rank) {
case 1:
result = new DataSetBuilder(rank, qube[0]);
break;
case 2:
result = new DataSetBuilder(rank, qube[0], qube[1]);
break;
case 3:
result = new DataSetBuilder(rank, qube[0], qube[1], qube[2]);
break;
case 4:
result = new DataSetBuilder(rank, qube[0], qube[1], qube[2], qube[3] );
break;
default:
throw new IllegalArgumentException("rank limit");
}
} else if (rank == qube.length + 1) {
switch (rank) {
case 1:
result = new DataSetBuilder(rank, 100);
break;
case 2:
result = new DataSetBuilder(rank, 100, qube[0]);
break;
case 3:
result = new DataSetBuilder(rank, 100, qube[0], qube[1]);
break;
case 4:
result = new DataSetBuilder(rank, 100, qube[0], qube[1], qube[2] );
break;
default:
throw new IllegalArgumentException("rank limit");
}
} else if (rank == qube.length + 2) {
switch (rank) {
case 2:
result = new DataSetBuilder(rank - 1, 100);
break;
case 3:
result = new DataSetBuilder(rank - 1, 100, qube[0]);
break;
case 4:
result = new DataSetBuilder(rank - 1, 100, qube[0], qube[1] );
break;
default:
throw new IllegalArgumentException("rank limit");
}
} else {
throw new IllegalArgumentException("rank and qube don't reconcile");
}
return result;
}
@Override
public void streamDescriptor(StreamDescriptor sd) throws StreamException {
logger.finer("stream descriptor");
try {
Element e = sd.getDomElement();
dsname = xpath.evaluate("//stream/@dataset_id", e);
if (dsname.length() == 0) {
Node stream = (Node) xpath.evaluate("//stream", e, XPathConstants.NODE);
if (stream != null) {
throw new StreamException("dataset_id attribute expected in the stream descriptor. Is this a qstream?");
} else {
throw new StreamException("dataset_id attribute expected in the stream descriptor. Expected to find stream.");
}
}
logger.log(Level.FINE, "got streamDescriptor with default dataset {0}", dsname);
} catch (XPathExpressionException ex) {
logger.log(Level.SEVERE, ex.getMessage(), ex);
}
}
/**
* parse the data found in the node. TODO: we don't have the Units yet, but we could and should use it to
* parse.
*
* @param vn
* @return
* @throws XPathExpressionException
*/
private DDataSet doInLine(Element vn) throws XPathExpressionException {
String svals = xpath.evaluate("@values", vn).trim();
if ( svals.equals("") ) {
svals= xpath.evaluate("@inline", vn).trim();
}
String sdims = xpath.evaluate("@length", vn);
int[] dims;
String[] ss = svals.split(",");
if (sdims == null || sdims.length() == 0) {
if (svals.length() == 0) {
dims = new int[0];
} else {
dims = new int[]{ss.length};
}
} else {
dims = Util.decodeArray(sdims);
}
int total = dims[0];
for (int i = 1; i < dims.length; i++) {
total *= dims[i];
}
double[] data;
if (!svals.trim().equals("")) { //TODO: length 0?
data = new double[total];
if (total != ss.length) {
throw new IllegalArgumentException("number of elements inline doesn't match length");
}
for (int j = 0; j < total; j++) {
data[j] = Double.parseDouble(ss[j]);
}
} else {
if (total > 0) {
throw new IllegalArgumentException("nonzero length but no inline elements");
}
data = new double[0];
}
DDataSet result = DDataSet.wrap(data, dims);
return result;
}
@Override
public void packetDescriptor(PacketDescriptor pd) throws StreamException {
try {
Element e = pd.getDomElement();
XPathExpression expr = xpath.compile("/packet/qdataset");
Object o = expr.evaluate(e, XPathConstants.NODESET);
NodeList nodes = (NodeList) o;
List<PlaneDescriptor> planes = pd.getPlanes();
if (planes.size() != nodes.getLength()) {
logger.log(Level.WARNING, "planes.size and nodes.getLength should be the same length, QDataSetStreamHandler line 254");
}
for (int i = 0; i < nodes.getLength(); i++) {
Element n = (Element) nodes.item(i);
String name = n.getAttribute("id");
logger.log(Level.FINER, "got packetDescriptor for {0}", name);
int rank = Integer.parseInt(n.getAttribute("rank"));
ranks.put( name, rank );
DataSetBuilder builder = null;
String sdims;
int[] dims;
String joinChildren; // join will be specified.
String joinParent;
boolean isInline = false;
joinParent = n.getAttribute("joinId");
NodeList values = (NodeList) xpath.evaluate("values", n, XPathConstants.NODESET);
NodeList bundles = null;
if (values.getLength() == 0) {
bundles = (NodeList) xpath.evaluate("bundle", n, XPathConstants.NODESET);
if (bundles.getLength() == 0) {
throw new IllegalArgumentException("no values node in " + n.getNodeName() + " " + n.getAttribute("id"));
} else {
logger.log(Level.FINER, "newBundle");
}
}
if (bundles != null) {
//nothing to do yet...
builder = new DataSetBuilder(0, 0);
builders.put(name, builder); // this is to hold properties.
String[] ss = planes.get(i).getBundles();
bundleDataSets.put(name, ss);
} else {
for (int iv = 0; iv < values.getLength(); iv++) {
Element vn = (Element) values.item(iv);
DDataSet inlineDs = null;
int index=-1;
if (vn.hasAttribute("values")) { // TODO: consider "inline"
inlineDs = doInLine(vn);
inlineDs.putProperty( QDataSet.UNITS, planes.get(i).getUnits() );
isInline = true;
if ( vn.hasAttribute("index") ) {
index= Integer.parseInt( vn.getAttribute("index") );
}
} else if ( vn.hasAttribute("inline")) { // TODO: consider "inline"
inlineDs = doInLine(vn);
isInline = true;
if ( vn.hasAttribute("index") ) {
index= Integer.parseInt( vn.getAttribute("index") );
}
} else if ( vn.hasAttribute("bundle") ) {
builder = new DataSetBuilder(1, 0);
builders.put(name, builder); // this is to hold properties.
String[] ss = planes.get(i).getBundles();
bundleDataSets.put(name, ss);
}
//index stuff--Ed W. thinks index should be implicit.
sdims = xpath.evaluate("@length", vn);
joinChildren = xpath.evaluate("@join", vn);
if (sdims == null) {
dims = new int[0];
} else {
dims = Util.decodeArray(sdims);
}
if (isInline && inlineDs != null && inlineDs.rank() < rank) { // I believe assert inlineDs!=null
if ( iv==0 && joinDataSets.containsKey(name) ) {
logger.log(Level.FINE, "resetting join dataset for name {0}", name);
joinDataSets.remove(name);
}
JoinDataSet join = getJoinDataSet(name,rank);
if ( index>=0 ) {
join.join(index,inlineDs);
} else {
join.join(inlineDs);
}
builder = new DataSetBuilder(0);
builders.put(name, builder); // rank 0 means the values were in line.
} else if (isInline && inlineDs != null && inlineDs.rank() == rank) {
builder = new DataSetBuilder(rank, inlineDs.length());
for (int j = 0; j < inlineDs.length(); j++) {
DDataSet slice = (DDataSet) inlineDs.slice(j);
builder.putValues(-1, slice, DataSetUtil.totalLength(slice));
builder.nextRecord();
}
builders.put(name, builder); // rank 0 means the values were in line.
} else if (joinChildren.length() > 0) {
getJoinDataSet(name,rank); // make sure it is allocated.
builder = new DataSetBuilder(1, 10);
builder.putProperty(BUILDER_JOIN_CHILDREN, joinChildren);
builders.put(name, builder); //
} else {
builder = builders.get(name);
if (builder == null) {
builder = createBuilder(rank, dims);
builders.put(name, builder);
if (!joinParent.equals("")) {
JoinDataSet parent = joinDataSets.get(joinParent);
String children = (String) parent.property(BUILDER_JOIN_CHILDREN);
if (children == null || children.length() == 0) {
children = name;
} else {
children = children + "," + name;
}
parent.putProperty(BUILDER_JOIN_CHILDREN, children);
}
} else {
JoinDataSet join;
if (joinParent.equals("")) {
join = joinDataSets.get(name); // old scheme
} else {
join = joinDataSets.get(joinParent);
}
if (join == null) { // /home/jbf/ct/hudson/data.backup/qds/aggregation.qds
logger.log(Level.FINE, "repeat of packet type for {0}, increasing rank.", name);
join = new JoinDataSet(rank + 1);
joinDataSets.put(name, join);
}
if ( !bundleDataSets.containsKey(name) ) {
builder.setDataSetResolver( getResolver() );
MutablePropertyDataSet mds = resolveProps(name, builder.getDataSet());
join.join(mds);
builder = createBuilder(rank, dims);
builders.put(name, builder);
}
}
}
}
}
NodeList odims = (NodeList) xpath.evaluate("properties[not(@index)]/property", n, XPathConstants.NODESET);
doProps(odims, builder);
if ( planes.get(i).getUnits() instanceof EnumerationUnits && builder!=null ) {
EnumerationUnits eu= (EnumerationUnits) planes.get(i).getUnits();
EnumerationUnits eun= (EnumerationUnits) builder.getProperties().get(QDataSet.UNITS);
if ( eu!=null && eun!=null ) {
for ( Entry<Integer,Datum> es: eu.getValues().entrySet() ) {
Datum d= es.getValue();
eun.createDatum( (int)d.doubleValue(eu), d.toString(), eu.getColor(d) );
}
}
}
odims = (NodeList) xpath.evaluate("properties[@index]/property", n, XPathConstants.NODESET);
doPropsIndex(odims, joinDataSets.get(name));
PlaneDescriptor planeDescriptor = planes.get(i);
planeDescriptor.setBuilder(builder);
// we no longer do the addPlane stuff here, since it is done at the source.
// TODO: a lot of work is done twice here, but this takes a trivial amount of time.
}
} catch (XPathExpressionException ex) {
logger.log(Level.SEVERE, ex.getMessage(), ex);
throw new RuntimeException(ex);
}
}
/**
* resolve properties that can be resolved, namely DEPENDNAME_0 which is a string linking two datasets.
*
* @param name the name of this dataset in the file.
* @param result the dataset we are resolving.
*/
private MutablePropertyDataSet resolveProps(String name, MutablePropertyDataSet result) {
// read datasets that need to be resolved.
for (int i = 0; i < QDataSet.MAX_RANK; i++) {
String s = (String) result.property("DEPENDNAME_" + i);
if (s != null) {
if (s.equals(name)) {
//result.putProperty( QDataSet.DEPEND_0, result ); // sometimes it's nice to look at Epoch[Epoch]
continue;
}
QDataSet dep0ds = getDataSetInternal(s);
if ( dep0ds==null ) {
logger.log(Level.WARNING, "unable to resolve property DEPENDNAME_{0}=\"{1}\"", new Object[]{i, s});
} else if (dep0ds.rank() == 1) {
result.putProperty("DEPEND_" + i, dep0ds);
} else if (dep0ds.rank()==2 && SemanticOps.isBins(dep0ds) ){
result.putProperty("DEPEND_" + i, dep0ds);
} else if (i > 0 && dep0ds.rank() == 2) {
result.putProperty("DEPEND_" + i, dep0ds);
} else {
//we're building DEPEND_0 as well, so this is resolved at the end.
//System.err.println("dropping DEPEND_0 for now");
}
}
}
// check for legacy behavior where string was used to refer to dataset.
for (int i = 0; i < QDataSet.MAX_RANK; i++) {
Object o = result.property("DEPEND_" + i);
if (o != null && o instanceof String) {
String s = (String) o;
logger.log(Level.WARNING, "QDataSetStreamHandler: still strings in DEPEND_{0}", i);
result.putProperty("DEPEND_" + i, getDataSetInternal(s));
}
}
for (int i = 0; i < QDataSet.MAX_RANK; i++) {
Object o = result.property("BUNDLE_" + i);
if (o instanceof String) { //TODO: still need to clean up messages here, maybe...
String s = (String) o;
result.putProperty("BUNDLE_" + i, getDataSetInternal(s));
}
}
for (int i = 0; i < QDataSet.MAX_PLANE_COUNT; i++) {
String propertyName= "PLANE_" + i;
Object o= result.property(propertyName);
if ( o instanceof String ) {
String s = (String) o;
result.putProperty( propertyName, getDataSetInternal(s) );
} else if ( o==null ) {
break;
}
}
String[] propertyNames= new String[] { "DELTA_MINUS", "DELTA_PLUS",
"BIN_MINUS", "BIN_PLUS",
"BIN_MIN", "BIN_MAX", "WEIGHTS"
};
for (String propertyName : propertyNames) {
Object o= result.property(propertyName);
if ( o instanceof String ) {
String s = (String) o;
result.putProperty( propertyName, getDataSetInternal(s) );
}
}
return result;
}
@Override
public void packet(PacketDescriptor pd, ByteBuffer data) throws StreamException {
logger.finer("stream packet");
if (readPackets) {
for (PlaneDescriptor planeDescriptor : pd.planes) {
DataSetBuilder builder = planeDescriptor.getBuilder();
if (planeDescriptor.getElements() > 1) {
DDataSet rec = DDataSet.createRank1(planeDescriptor.getElements());
for (int ii = 0; ii < planeDescriptor.getElements(); ii++) {
rec.putValue(ii, planeDescriptor.getType().read(data));
}
if (pd.isStream() == false) {
if (planeDescriptor.getRank() > 1) {
throw new IllegalArgumentException("non-streaming and rank>1 not supported");
}
for (int i = 0; i < rec.length(); i++) {
builder.putValue(-1, rec.value(i));
builder.nextRecord();
}
} else {
builder.putValues(-1, rec, planeDescriptor.getElements()); // aliasing okay
builder.nextRecord();
}
} else {
TransferType tt = planeDescriptor.getType();
if (tt == null) {
logger.severe("here planeDescriptor.getType() is null");
}
builder.putValue(-1, planeDescriptor.getType().read(data));
builder.nextRecord();
}
}
}
}
@Override
public void streamClosed(StreamDescriptor sd) throws StreamException {
logger.finer("stream closed");
}
@Override
public void streamComment(StreamComment se) throws StreamException {
logger.log(Level.FINER, "stream comment: {0}", se.getMessage());
}
@Override
public void streamException(StreamException se) throws StreamException {
logger.log(Level.FINER, "stream exception: {0}", se.getMessage());
}
private DataSetBuilder.DataSetResolver getResolver() {
return new DataSetBuilder.DataSetResolver() {
@Override
public QDataSet resolve( String name ) {
QDataSet result= joinDataSets.get(name);
if ( result==null ) {
DataSetBuilder builder= builders.get(name);
if ( builder!=null ) {
result= builder.getDataSet();
}
}
return result;
}
};
}
/**
* get the JoinDataSet into which we will store datasets. Start
* a new one if the dataset does not exist, or return the one we are working
* on.
* @param name name of the join
* @param rank the requested or expected rank.
*/
private JoinDataSet getJoinDataSet( String name, int rank ) {
JoinDataSet join = joinDataSets.get(name);
if (join == null) {
join = new JoinDataSet(rank);
joinDataSets.put(name, join);
} else {
if ( join.rank()!=rank ) {
throw new IllegalArgumentException("rank mismatch");
}
}
return join;
}
/**
* return the dataset from the stream.
*
* @param name the name of the dataset to retrieve.
* @return the dataset
*/
public QDataSet getDataSet(String name) {
QDataSet result= getDataSetInternal(name);
if ( result==null ) {
throw new IllegalArgumentException("No such dataset \"" + name + "\"");
}
Integer rank= ranks.get(name);
if ( rank!=result.rank() && isFlattenableJoin(result) ) {
logger.log(Level.FINE, "flattening join for {0}: {1}", new Object[]{name, result});
result= flattenJoin(result);
}
return result;
}
/**
* return the default dataset for the stream. The default dataset is
* identified in the stream packet descriptor dataset_id attribute.
*
* @return the default dataset
*/
public QDataSet getDataSet() {
return getDataSet(dsname);
}
/**
* return the dataset from the stream
*
* @param name the name of the dataset to retrieve.
* @return the dataset or null if no such dataset exists.
*/
private QDataSet getDataSetInternal(String name) {
logger.log(Level.FINE, "getDataSet({0})", name);
DataSetBuilder builder = builders.get(name);
String[] sbds = bundleDataSets.get(name);
JoinDataSet join = joinDataSets.get(name);
if (builder == null && sbds == null) {
logger.log(Level.INFO, "no such dataset: {0}", name);
return null;
}
MutablePropertyDataSet result;
MutablePropertyDataSet sliceDs = null;
if (join != null && sbds==null ) {
String joinChild = (String) join.property(BUILDER_JOIN_CHILDREN);
join = JoinDataSet.copy(join);
if (builder != null && builder.rank() > 0) {
builder.setDataSetResolver( getResolver() );
sliceDs = builder.getDataSet();
List<QDataSet> childDataSets = null;
if ( sliceDs.property(BUILDER_JOIN_CHILDREN) != null) {
if (joinChild == null) {
joinChild = (String) sliceDs.property(BUILDER_JOIN_CHILDREN);
}
String[] children = joinChild.split(",");
childDataSets = new ArrayList();
for (String children1 : children) {
DataSetBuilder childBuilder = builders.get(children1);
if (childBuilder != null) {
childBuilder.setDataSetResolver( getResolver() );
MutablePropertyDataSet sliceDs1 = childBuilder.getDataSet();
resolveProps(null, sliceDs1);
childDataSets.add(sliceDs1);
logger.log(Level.FINER, "child: {0}", sliceDs1.toString());
} else {
logger.log(Level.WARNING, "missing child: {0}", children1);
}
}
}
if (childDataSets != null) {
for (QDataSet child : childDataSets) {
join.join(child);
}
}
DataSetUtil.putProperties(builder.getProperties(), join);
if ( sliceDs.length() > 0) {
resolveProps(null, sliceDs);
logger.fine("aggregation has one last dataset to append");
join.join(sliceDs);
}
} else {
assert builder != null;
DataSetUtil.putProperties(builder.getProperties(), join);
sliceDs = (MutablePropertyDataSet) join.slice(join.length() - 1); //wha?? when do we use this?
}
result = join;
} else if (sbds != null) {
BundleDataSet bds = new BundleDataSet();
for (String sbd : sbds) {
bds.bundle(getDataSet(sbd));
}
result = bds;
} else {
assert builder != null;
builder.setDataSetResolver( getResolver() );
result = builder.getDataSet();
}
if (join != null) {
if (sliceDs != null) {
resolveProps(name, sliceDs); //TODO: this technically breaks things, because we cannot call getDataSet again
}
} else {
resolveProps(name, result);
}
return result;
}
/**
* set this is false if you just want to look at the empty dataset metadata.
*
* @param val
*/
public void setReadPackets(boolean val) {
this.readPackets = val;
}
/**
* if true, then packets are interpreted.
*
* @return
*/
public boolean getReadPackets() {
return this.readPackets;
}
private void doProps(NodeList odims, DataSetBuilder builder) {
for (int j = 0; j < odims.getLength(); j++) {
Element n2 = (Element) odims.item(j);
String pname = n2.getAttribute("name");
String svalue;
if (n2.hasAttribute("value")) {
svalue = n2.getAttribute("value");
} else {
svalue = n2.getTextContent();
}
Element evalue = null;
String stype;
if (n2.hasAttribute("type")) {
stype = n2.getAttribute("type");
} else {
evalue = Util.singletonChildElement(n2);
stype = evalue.getTagName();
}
if (stype.equals("qdataset")) {
if (pname.equals(QDataSet.DELTA_MINUS) || pname.equals(QDataSet.DELTA_PLUS)) {
logger.warning("skipping DELTA_MINUS and DELTA_PLUS because bug");
builder.putProperty(pname, svalue);
builder.putUnresolvedProperty(DataSetBuilder.UNRESOLVED_PROP_QDATASET,pname,svalue);
}
if (pname.matches("DEPEND_\\d+")) {
String si = pname.substring(7);
builder.putProperty("DEPENDNAME_" + si, svalue);
} else {
builder.putProperty(pname, svalue);
builder.putUnresolvedProperty(DataSetBuilder.UNRESOLVED_PROP_QDATASET,pname,svalue);
}
} else {
SerializeDelegate delegate = SerializeRegistry.getByName(stype);
if (delegate == null) {
logger.log(Level.SEVERE, "!!! No delegate found for \"{0}\"", stype); // chris and I didn't see this invisible message
continue;
}
Object oval;
try {
if (evalue != null && delegate instanceof XMLSerializeDelegate) {
oval = ((XMLSerializeDelegate) delegate).xmlParse(evalue);
} else {
if ( stype.equals("enumerationUnit") ) {
oval= enumerationUnitsSerializeDelegate.parse( stype, svalue );
} else {
oval = delegate.parse(stype, svalue);
}
}
builder.putProperty(pname, oval);
} catch (ParseException ex) {
logger.log(Level.SEVERE, ex.getMessage(), ex);
}
}
}
}
private static void doPropsIndex(NodeList odims, JoinDataSet join) {
for (int j = 0; j < odims.getLength(); j++) {
Element n2 = (Element) odims.item(j);
String sidx = ((Element) n2.getParentNode()).getAttribute("index");
int index = Integer.parseInt(sidx);
String pname = n2.getAttribute("name");
String svalue;
if (n2.hasAttribute("value")) {
svalue = n2.getAttribute("value");
} else {
svalue = n2.getTextContent();
}
Element evalue = null;
String stype;
if (n2.hasAttribute("type")) {
stype = n2.getAttribute("type");
} else {
evalue = Util.singletonChildElement(n2);
stype = evalue.getTagName();
}
if (stype.equals("qdataset")) {
join.putProperty(pname, index, svalue);
} else {
SerializeDelegate delegate = SerializeRegistry.getByName(stype);
if (delegate == null) {
logger.log(Level.SEVERE, "no delegate found for \"{0}\"", stype);
continue;
}
Object oval;
try {
if (evalue != null && delegate instanceof XMLSerializeDelegate) {
oval = ((XMLSerializeDelegate) delegate).xmlParse(evalue);
} else {
oval = delegate.parse(stype, svalue);
}
join.putProperty(pname, index, oval);
} catch (ParseException ex) {
logger.log(Level.SEVERE, ex.getMessage(), ex);
}
}
}
}
/**
* If the dataset is a join of appendable datasets, then we can append them to reduce the rank by 1 and
* make one long time series. These datasets should be equivalent, however most of the system doesn't
* implement this (and probably never will). So this is a bit of a kludge, where I don't want to flatten a
* dataset automatically, but we probably want to.
*
* @param ds a join dataset of rank 2 or rank 3.
* @return true if the data can be joined.
*/
public static boolean isFlattenableJoin(QDataSet ds) {
if ( ds.rank() == 2 && ds.property(QDataSet.DEPEND_0)==null && ds.property(QDataSet.BUNDLE_1)==null && ds.property(QDataSet.BINS_1)==null ) {
return true;
} else if (ds.rank() == 3 && ds.length() > 0 && ds.property(QDataSet.DEPEND_0) == null && ds.property(QDataSet.DEPENDNAME_0) != null) {
QDataSet dep1 = (QDataSet) ds.slice(0).property(QDataSet.DEPEND_1);
for (int i = 1; i < ds.length(); i++) {
if (dep1 == null) {
if (ds.rank() == 3) { // join of vector datasets.
if (ds.length(0, 0) != ds.length(i, 0)) {
return false;
}
} else { // ds.rank()==2, old code.
if (ds.length(0) != ds.length(i)) {
return false;
}
}
} else {
QDataSet dep1t = (QDataSet) ds.slice(i).property(QDataSet.DEPEND_1);
if (!Ops.equivalent(dep1, dep1t)) {
return false;
}
}
}
return true;
}
return false;
}
/**
* since an appended series of rank 1 datasets will return as a rank 2 join, this utility provides a
* standard place to flatten it. This will also flatten DEPENDNAME_0.
*
* @param ds rank 2 or 3 join dataset.
* @return rank 1 or 2 dataset.
*/
public MutablePropertyDataSet flattenJoin(QDataSet ds) {
int len = 0;
if ( ds.rank()<2 ) throw new IllegalArgumentException("rank should be > 2" );
for (int i = 0; i < ds.length(); i++) {
len += ds.length(i);
}
ArrayDataSet result = ArrayDataSet.maybeCopy(ds.slice(0));
if ( result.isImmutable() ) result= ArrayDataSet.copy(result);
result.grow(len);
for (int i = 1; i < ds.length(); i++) {
result.append(ArrayDataSet.maybeCopy(ds.slice(i)));
}
String s = (String) ds.property(QDataSet.DEPENDNAME_0);
if (s != null) {
MutablePropertyDataSet dep0 = (MutablePropertyDataSet) getDataSet(s);
//dep0 = (MutablePropertyDataSet) flattenJoin(dep0); //getDataSet has already flattened it.
dep0.putProperty(QDataSet.TYPICAL_MIN, null); // remove so we don't just use MIN and MAX from one dataset.
dep0.putProperty(QDataSet.TYPICAL_MAX, null);
((MutablePropertyDataSet) result).putProperty(QDataSet.DEPEND_0, dep0);
((MutablePropertyDataSet) result).putProperty(QDataSet.DEPENDNAME_0, null);
}
return result;
}
}