001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.reflect.Array; 017import java.text.Format; 018import java.util.ArrayList; 019import java.util.Arrays; 020import java.util.List; 021import java.util.concurrent.ConcurrentHashMap; 022import java.util.concurrent.ConcurrentMap; 023 024import org.eclipse.january.DatasetException; 025import org.eclipse.january.IMonitor; 026import org.eclipse.january.MetadataException; 027import org.eclipse.january.metadata.ErrorMetadata; 028import org.eclipse.january.metadata.MetadataFactory; 029import org.eclipse.january.metadata.MetadataType; 030import org.eclipse.january.metadata.StatisticsMetadata; 031import org.eclipse.january.metadata.internal.ErrorMetadataImpl; 032import org.eclipse.january.metadata.internal.StatisticsMetadataImpl; 033 034/** 035 * Generic container class for data 036 * <p/> 037 * Each subclass has an array of primitive types, elements of this array are grouped or 038 * compounded to make items 039 * <p/> 040 * Data items can be boolean, integer, float, complex float, vector float, etc 041 */ 042public abstract class AbstractDataset extends LazyDatasetBase implements Dataset { 043 // pin UID to base class 044 private static final long serialVersionUID = Dataset.serialVersionUID; 045 046 protected int size; // number of items 047 048 transient protected AbstractDataset base; // is null when not a view 049 protected int[] stride; // can be null for row-major, contiguous datasets 050 protected int offset; 051 052 /** 053 * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as 054 * wanted 055 */ 056 protected Serializable odata = null; 057 058 /** 059 * Set aliased data as base data 060 */ 061 abstract protected void setData(); 062 063 /** 064 * Constructor required for serialisation. 065 */ 066 public AbstractDataset() { 067 } 068 069 @Override 070 public synchronized Dataset synchronizedCopy() { 071 return clone(); 072 } 073 074 @Override 075 public boolean equals(Object obj) { 076 if (this == obj) { 077 return true; 078 } 079 if (obj == null) { 080 return false; 081 } 082 if (!getClass().equals(obj.getClass())) { 083 if (getRank() == 0) // for zero-rank datasets 084 return obj.equals(getObjectAbs(offset)); 085 return false; 086 } 087 088 Dataset other = (Dataset) obj; 089 if (getElementsPerItem() != other.getElementsPerItem()) 090 return false; 091 if (size != other.getSize()) 092 return false; 093 if (!Arrays.equals(shape, other.getShapeRef())) { 094 return false; 095 } 096 097 return true; 098 } 099 100 @Override 101 public int hashCode() { 102 return getStats().getHash(shape); 103 } 104 105 @Override 106 abstract public AbstractDataset clone(); 107 108 protected Format stringFormat = null; 109 110 @Override 111 public void setStringFormat(Format format) { 112 stringFormat = format; 113 } 114 115 @Override 116 public Dataset copy(final int dtype) { 117 if (getDType() == dtype) { 118 return clone(); 119 } 120 return DatasetUtils.copy(this, dtype); 121 } 122 123 @Override 124 public <T extends Dataset> T copy(Class<T> clazz) { 125 return DatasetUtils.copy(clazz, this); 126 } 127 128 @Override 129 public Dataset cast(final int dtype) { 130 if (getDType() == dtype) { 131 return this; 132 } 133 return DatasetUtils.cast(this, dtype); 134 } 135 136 @Override 137 public <T extends Dataset> T cast(Class<T> clazz) { 138 return DatasetUtils.cast(clazz, this); 139 } 140 141 @Override 142 public Dataset cast(final boolean repeat, final int dtype, final int isize) { 143 if (getDType() == dtype && getElementsPerItem() == isize) { 144 return this; 145 } 146 return DatasetUtils.cast(this, repeat, dtype, isize); 147 } 148 149 @Override 150 abstract public AbstractDataset getView(boolean deepCopyMetadata); 151 152 /** 153 * Copy fields from original to view 154 * @param orig 155 * @param view 156 * @param clone if true, then clone everything but bulk data 157 * @param cloneMetadata if true, clone metadata 158 */ 159 protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) { 160 view.name = orig.getName(); 161 view.size = orig.getSize(); 162 view.odata = orig.getBuffer(); 163 view.offset = orig.getOffset(); 164 view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null; 165 166 if (clone) { 167 view.shape = orig.getShape(); 168 view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ? 169 ((AbstractDataset) orig).stride.clone() : null; 170 } else { 171 view.shape = orig.getShapeRef(); 172 view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null; 173 } 174 175 view.metadata = getMetadataMap(orig, cloneMetadata); 176 int odtype = orig.getDType(); 177 int vdtype = view.getDType(); 178 if (odtype != vdtype) { 179 view.setDirty(); 180 } 181 } 182 183 /** 184 * @since 2.0 185 */ 186 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(Dataset a, boolean clone) { 187 if (a == null) 188 return null; 189 190 List<MetadataType> all = null; 191 try { 192 all = a.getMetadata(null); 193 } catch (Exception e) { 194 } 195 if (all == null) 196 return null; 197 198 ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 199 200 for (MetadataType m : all) { 201 if (m == null) { 202 continue; 203 } 204 Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass()); 205 List<MetadataType> l = map.get(c); 206 if (l == null) { 207 l = new ArrayList<MetadataType>(); 208 map.put(c, l); 209 } 210 if (clone) 211 m = m.clone(); 212 l.add(m); 213 } 214 return map; 215 } 216 217 @Override 218 public IntegerDataset getIndices() { 219 final IntegerDataset ret = DatasetUtils.indices(shape); 220 if (getName() != null) { 221 ret.setName("Indices of " + getName()); 222 } 223 return ret; 224 } 225 226 @Override 227 public Dataset getTransposedView(int... axes) { 228 axes = checkPermutatedAxes(shape, axes); 229 230 AbstractDataset t = getView(true); 231 if (axes == null || getRank() == 1) 232 return t; 233 234 int rank = shape.length; 235 int[] tstride = new int[rank]; 236 int[] toffset = new int[1]; 237 int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset); 238 int[] nstride = new int[rank]; 239 for (int i = 0; i < rank; i++) { 240 final int ax = axes[i]; 241 nstride[i] = tstride[ax]; 242 nshape[i] = shape[ax]; 243 } 244 t.shape = nshape; 245 t.stride = nstride; 246 t.offset = toffset[0]; 247 t.base = this; 248 t.setDirty(); 249 t.transposeMetadata(axes); 250 return t; 251 } 252 253 @Override 254 public Dataset transpose(int... axes) { 255 Dataset t = getTransposedView(axes); 256 return t == null ? clone() : t.clone(); 257 } 258 259 @Override 260 public Dataset swapAxes(int axis1, int axis2) { 261 int rank = shape.length; 262 axis1 = ShapeUtils.checkAxis(rank, axis1); 263 axis2 = ShapeUtils.checkAxis(rank, axis2); 264 265 if (rank == 1 || axis1 == axis2) { 266 return this; 267 } 268 269 int[] axes = new int[rank]; 270 for (int i = 0; i < rank; i++) { 271 axes[i] = i; 272 } 273 274 axes[axis1] = axis2; 275 axes[axis2] = axis1; 276 return getTransposedView(axes); 277 } 278 279 private boolean isContiguous() { 280 if (stride == null) 281 return true; 282 283 if (offset != 0) 284 return false; 285 286 int s = getElementsPerItem(); 287 for (int j = getRank() - 1; j >= 0; j--) { 288 if (stride[j] != s) { 289 return false; 290 } 291 s *= shape[j]; 292 } 293 294 return true; 295 } 296 297 @Override 298 public Dataset flatten() { 299 if (!isContiguous()) { // need to make a copy if not contiguous 300 return clone().flatten(); 301 } 302 return reshape(size); 303 } 304 305 /** 306 * Fill dataset from object at depth dimension 307 * @param obj 308 * @param depth 309 * @param pos position 310 */ 311 protected void fillData(Object obj, final int depth, final int[] pos) { 312 if (obj == null) { 313 int dtype = getDType(); 314 if (dtype == FLOAT32) 315 set(Float.NaN, pos); 316 else if (dtype == FLOAT64) 317 set(Double.NaN, pos); 318 return; 319 } 320 321 Class<?> clazz = obj.getClass(); 322 if (obj instanceof List<?>) { 323 List<?> jl = (List<?>) obj; 324 int l = jl.size(); 325 for (int i = 0; i < l; i++) { 326 Object lo = jl.get(i); 327 fillData(lo, depth + 1, pos); 328 pos[depth]++; 329 } 330 pos[depth] = 0; 331 } else if (clazz.isArray()) { 332 int l = Array.getLength(obj); 333 if (clazz.equals(odata.getClass())) { 334 System.arraycopy(obj, 0, odata, get1DIndex(pos), l); 335 } else if (clazz.getComponentType().isPrimitive()) { 336 for (int i = 0; i < l; i++) { 337 set(Array.get(obj, i), pos); 338 pos[depth]++; 339 } 340 pos[depth] = 0; 341 } else { 342 for (int i = 0; i < l; i++) { 343 fillData(Array.get(obj, i), depth + 1, pos); 344 pos[depth]++; 345 } 346 pos[depth] = 0; 347 } 348 } else if (obj instanceof IDataset) { 349 boolean[] a = new boolean[shape.length]; 350 for (int i = depth; i < a.length; i++) 351 a[i] = true; 352 setSlice(obj, getSliceIteratorFromAxes(pos, a)); 353 } else { 354 set(obj, pos); 355 } 356 } 357 358 @Override 359 public IndexIterator getIterator(final boolean withPosition) { 360 if (stride != null) { 361 return base.getSize() == 1 ? (withPosition ? new PositionIterator(offset, shape) : 362 new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset); 363 } 364 if (shape == null) { 365 return new NullIterator(shape, shape); 366 } 367 368 return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size); 369 } 370 371 @Override 372 public IndexIterator getIterator() { 373 return getIterator(false); 374 } 375 376 @Override 377 public PositionIterator getPositionIterator(final int... axes) { 378 return new PositionIterator(shape, axes); 379 } 380 381 @Override 382 public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) { 383 return getSliceIterator(new SliceND(shape, start, stop, step)); 384 } 385 386 /** 387 * @param slice 388 * @return an slice iterator that operates like an IndexIterator 389 */ 390 public IndexIterator getSliceIterator(SliceND slice) { 391 if (ShapeUtils.calcLongSize(slice.getShape()) == 0) { 392 return new NullIterator(shape, slice.getShape()); 393 } 394 if (stride != null) { 395 return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice); 396 } 397 return new SliceIterator(shape, size, slice); 398 } 399 400 @Override 401 public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) { 402 int rank = shape.length; 403 int[] start; 404 int[] stop = new int[rank]; 405 int[] step = new int[rank]; 406 407 if (pos == null) { 408 start = new int[rank]; 409 } else if (pos.length == rank) { 410 start = pos.clone(); 411 } else { 412 throw new IllegalArgumentException("pos array length is not equal to rank of dataset"); 413 } 414 if (axes == null) { 415 axes = new boolean[rank]; 416 Arrays.fill(axes, true); 417 } else if (axes.length != rank) { 418 throw new IllegalArgumentException("axes array length is not equal to rank of dataset"); 419 } 420 421 for (int i = 0; i < rank; i++) { 422 if (axes[i]) { 423 stop[i] = shape[i]; 424 } else { 425 stop[i] = start[i] + 1; 426 } 427 step[i] = 1; 428 } 429 return (SliceIterator) getSliceIterator(start, stop, step); 430 } 431 432 @Override 433 public BooleanIterator getBooleanIterator(Dataset choice) { 434 return getBooleanIterator(choice, true); 435 } 436 437 @Override 438 public BooleanIterator getBooleanIterator(Dataset choice, boolean value) { 439 return BooleanIterator.createIterator(value, this, choice, this); 440 } 441 442 @Override 443 public Dataset getByBoolean(Dataset selection) { 444 checkCompatibility(selection); 445 446 final int length = ((Number) selection.sum()).intValue(); 447 final int is = getElementsPerItem(); 448 Dataset r = DatasetFactory.zeros(is, getClass(), length); 449 BooleanIterator biter = getBooleanIterator(selection); 450 451 int i = 0; 452 while (biter.hasNext()) { 453 r.setObjectAbs(i, getObjectAbs(biter.index)); 454 i += is; 455 } 456 return r; 457 } 458 459 @Override 460 public Dataset getBy1DIndex(IntegerDataset index) { 461 final int is = getElementsPerItem(); 462 final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShape()); 463 final IntegerIterator iter = new IntegerIterator(index, size, is); 464 465 int i = 0; 466 while (iter.hasNext()) { 467 r.setObjectAbs(i, getObjectAbs(iter.index)); 468 i += is; 469 } 470 return r; 471 } 472 473 @Override 474 public Dataset getByIndexes(final Object... indexes) { 475 final IntegersIterator iter = new IntegersIterator(shape, indexes); 476 final int is = getElementsPerItem(); 477 final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape()); 478 479 final int[] pos = iter.getPos(); 480 int i = 0; 481 while (iter.hasNext()) { 482 r.setObjectAbs(i, getObject(pos)); 483 i += is; 484 } 485 return r; 486 } 487 488 @Override 489 public Class<?> getElementClass() { 490 return DTypeUtils.getElementClass(getDType()); 491 } 492 493 @Override 494 public boolean hasFloatingPointElements() { 495 Class<?> cls = getElementClass(); 496 return cls == Float.class || cls == Double.class; 497 } 498 499 @Override 500 public int getElementsPerItem() { 501 return DTypeUtils.getElementsPerItem(getDType()); 502 } 503 504 @Override 505 public int getItemBytes() { 506 return DTypeUtils.getItemBytes(getDType(), getElementsPerItem()); 507 } 508 509 @Override 510 public String getName() { 511 return name; 512 } 513 514 @Override 515 public void setName(final String name) { 516 this.name = name; 517 } 518 519 @Override 520 public int getSize() { 521 return size; 522 } 523 524 @Override 525 public int[] getShape() { 526 // make a copy of the dimensions data, and put that out 527 if (shape == null) { 528 logger.warn("Shape is null!!!"); 529 return new int[] {}; 530 } 531 return shape.clone(); 532 } 533 534 @Override 535 public int getRank() { 536 return shape == null ? 0 : shape.length; 537 } 538 539 @Override 540 public int getNbytes() { 541 return getSize() * getItemBytes(); 542 } 543 544 /** 545 * Check for -1 placeholder in shape and replace if necessary 546 * @param shape 547 * @param size 548 */ 549 private void checkShape(int[] shape, int size) { 550 int rank = shape.length; 551 int found = -1; 552 int nsize = 1; 553 for (int i = 0; i < rank; i++) { 554 int d = shape[i]; 555 if (d == -1) { 556 if (found == -1) { 557 found = i; 558 } else { 559 logger.error("Can only have one -1 placeholder in shape"); 560 throw new IllegalArgumentException("Can only have one -1 placeholder in shape"); 561 } 562 } else { 563 nsize *= d; 564 } 565 } 566 if (found >= 0) { 567 shape[found] = size/nsize; 568 } else if (nsize != size && !(rank == 0 && size == 0)) { 569 logger.error("New shape is not same size as old shape"); 570 throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape)); 571 } 572 } 573 574 @Override 575 public void setShape(final int... shape) { 576 int[] nshape = shape.clone(); 577 checkShape(nshape, size); 578 if (Arrays.equals(this.shape, nshape)) { 579 return; 580 } 581 582 if (stride != null) { 583 // the only compatible shapes are ones where new dimensions are factors of old dimensions 584 // or are combined adjacent old dimensions 585 int[] oshape = this.shape; 586 int orank = oshape.length; 587 int nrank = nshape.length; 588 int diff = nrank - orank; 589 int[] nstride = new int[nrank]; 590 boolean ones = true; 591 // work forwards for broadcasting cases 592 for (int i = 0, j = 0; i < orank || j < nrank;) { 593 if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) { 594 nstride[j++] = stride[i++]; 595 } else if (j < nrank && nshape[j] == 1) { 596 nstride[j++] = 0; 597 } else if (i < orank && oshape[i] == 1) { 598 i++; 599 } else { 600 if (j < nrank) 601 j++; 602 if (i < orank) 603 i++; 604 ones = false; 605 } 606 } 607 if (!ones) { // not just ones differ in shapes 608 int[] ostride = stride; 609 int ob = 0; 610 int oe = 1; 611 int nb = 0; 612 int ne = 1; 613 while (ob < orank && nb < nrank) { 614 int ol = oshape[ob]; 615 int nl = nshape[nb]; 616 617 if (nl < ol) { // find group of shape dimensions that form common size 618 do { // case where new shape spreads single dimension over several dimensions 619 if (ne == nrank) { 620 break; 621 } 622 nl *= nshape[ne++]; 623 } while (nl < ol); 624 if (nl != ol) { 625 logger.error("Subshape is incompatible with single dimension"); 626 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 627 } 628 int on = ne - 1; 629 while (nshape[on] == 1) { 630 on--; 631 } 632 633 nstride[on] = ostride[ob]; 634 for (int n = on - 1; n >= nb; n--) { 635 if (nshape[n] == 1) 636 continue; 637 638 nstride[n] = nshape[on] * nstride[on]; 639 on = n; 640 } 641 } else if (ol < nl) { 642 do { // case where new shape combines several dimensions into one dimension 643 if (oe == orank) { 644 break; 645 } 646 ol *= oshape[oe++]; 647 } while (ol < nl); 648 if (nl != ol) { 649 logger.error("Single dimension is incompatible with subshape"); 650 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 651 } 652 653 int oo = oe - 1; 654 while (oshape[oo] == 1) { 655 oo--; 656 } 657 int os = ostride[oo]; 658 for (int o = oo - 1; o >= ob; o--) { 659 if (oshape[o] == 1) 660 continue; 661 if (ostride[o] != oshape[oo] * ostride[oo]) { 662 logger.error("Subshape cannot be a non-contiguous view"); 663 throw new IllegalArgumentException("Subshape cannot be a non-contiguous view"); 664 } 665 oo = o; 666 } 667 nstride[nb] = os; 668 } else { 669 nstride[nb] = ostride[ob]; 670 } 671 672 ob = oe++; 673 nb = ne++; 674 } 675 } 676 677 stride = nstride; 678 } 679 680 setDirty(); 681 if (this.shape != null) { 682 reshapeMetadata(this.shape, nshape); 683 } 684 this.shape = nshape; 685 } 686 687 @Override 688 public int[] getShapeRef() { 689 return shape; 690 } 691 692 @Override 693 public int getOffset() { 694 return offset; 695 } 696 697 @Override 698 public int[] getStrides() { 699 return stride; 700 } 701 702 @Override 703 public Serializable getBuffer() { 704 return odata; 705 } 706 707 @Override 708 public void overrideInternal(Serializable buffer, int... shape) { 709 if (buffer != null) { 710 odata = buffer; 711 setData(); 712 setDirty(); 713 } 714 715 if (shape != null) { 716 this.shape = shape.clone(); 717 size = ShapeUtils.calcSize(this.shape); 718 } 719 } 720 721 /** 722 * Create a stride array from dataset 723 * @param a dataset 724 * @param offset output offset 725 * @return new strides 726 */ 727 public static int[] createStrides(Dataset a, final int[] offset) { 728 return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset); 729 } 730 731 /** 732 * Create a stride array from dataset 733 * @param isize 734 * @param shape 735 * @param oStride original stride 736 * @param oOffset original offset (only used if there is an original stride) 737 * @param offset output offset 738 * @return new strides 739 */ 740 public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) { 741 int rank = shape.length; 742 final int[] stride; 743 if (oStride == null) { 744 offset[0] = 0; 745 stride = new int[rank]; 746 int s = isize; 747 for (int j = rank - 1; j >= 0; j--) { 748 stride[j] = s; 749 s *= shape[j]; 750 } 751 } else { 752 offset[0] = oOffset; 753 stride = oStride.clone(); 754 } 755 return stride; 756 } 757 758 /** 759 * Create a stride array from slice information and a dataset 760 * @param slice 761 * @param a dataset 762 * @param stride output stride 763 * @param offset output offset 764 * @return new shape 765 */ 766 public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) { 767 return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset); 768 } 769 770 /** 771 * Create a stride array from slice and dataset information 772 * @param slice 773 * @param isize 774 * @param shape 775 * @param oStride original stride 776 * @param oOffset original offset (only used if there is an original stride) 777 * @param stride output stride 778 * @param offset output offset 779 * @return new shape 780 */ 781 public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) { 782 int[] lstart = slice.getStart(); 783 int[] lstep = slice.getStep(); 784 int[] newShape = slice.getShape(); 785 int rank = shape.length; 786 787 if (oStride == null) { 788 int s = isize; 789 offset[0] = 0; 790 for (int j = rank - 1; j >= 0; j--) { 791 stride[j] = s * lstep[j]; 792 offset[0] += s * lstart[j]; 793 s *= shape[j]; 794 } 795 } else { 796 offset[0] = oOffset; 797 for (int j = 0; j < rank; j++) { 798 int s = oStride[j]; 799 stride[j] = lstep[j] * s; 800 offset[0] += lstart[j] * s; 801 } 802 } 803 804 return newShape; 805 } 806 807 @Override 808 public Dataset getBroadcastView(int... broadcastShape) { 809 AbstractDataset view = getView(true); 810 811 if (!Arrays.equals(shape, broadcastShape)) { 812 List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape); 813 view.setShape(nShapes.get(0)); 814 view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape); 815 view.base = this; 816 view.shape = broadcastShape.clone(); 817 view.size = ShapeUtils.calcSize(broadcastShape); 818 if (view.name == null || view.name.isEmpty()) { 819 view.name = "Broadcast from " + Arrays.toString(shape); 820 } else { 821 view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape); 822 } 823 } 824 return view; 825 } 826 827 @Override 828 public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) { 829 return getSliceView(new SliceND(shape, start, stop, step)); 830 } 831 832 @Override 833 public Dataset getSliceView(Slice... slice) { 834 if (slice == null || slice.length == 0) { 835 return getView(true); 836 } 837 838 return getSliceView(new SliceND(shape, slice)); 839 } 840 841 /** 842 * Get a slice of the dataset. The returned dataset is a view on a selection of items 843 * @param slice 844 * @return slice view 845 */ 846 @Override 847 public Dataset getSliceView(SliceND slice) { 848 if (slice.isAll()) { 849 return getView(true); 850 } 851 852 final int rank = shape.length; 853 int[] sStride = new int[rank]; 854 int[] sOffset = new int[1]; 855 856 int[] sShape = createStrides(slice, this, sStride, sOffset); 857 858 AbstractDataset s = getView(false); 859 s.shape = sShape; 860 s.size = ShapeUtils.calcSize(sShape); 861 s.stride = sStride; 862 s.offset = sOffset[0]; 863 s.base = this; 864 865 s.metadata = copyMetadata(); 866 s.sliceMetadata(true, slice); 867 868 s.setDirty(); 869 s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE); 870 871 return s; 872 } 873 874 /** 875 * Get flattened view index of given position 876 * @param pos 877 * the integer array specifying the n-D position 878 * @return the index on the flattened dataset 879 */ 880 private int getFlat1DIndex(final int[] pos) { 881 final int imax = pos.length; 882 if (imax == 0) { 883 return 0; 884 } 885 886 return get1DIndexFromShape(pos); 887 } 888 889 /** 890 * @since 2.0 891 */ 892 protected int getFirst1DIndex() { 893 if (shape == null) { 894 throw new IllegalArgumentException("Cannot find an index from a null shape"); 895 } 896 return stride == null ? 0 : offset; 897 } 898 899 @Override 900 public int get1DIndex(final int... n) { 901 if (n.length == 0 && shape.length == 0) 902 return offset; 903 904 return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n); 905 } 906 907 private static void throwAIOOBException(int i, int s, int d) { 908 throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s 909 + "] in dimension " + d); 910 } 911 912 /** 913 * @param i 914 * @return the index on the data array corresponding to that location 915 */ 916 protected int get1DIndex(int i) { 917 if (shape == null) { 918 throw new IllegalArgumentException("Cannot find an index from a null shape"); 919 } 920 if (shape.length > 1) { 921 logger.error("This dataset is not 1D but was addressed as such"); 922 throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such"); 923 } 924 if (i < 0) { 925 i += shape[0]; 926 } 927 if (i < 0 || i >= shape[0]) { 928 throwAIOOBException(i, shape[0], 0); 929 } 930 return stride == null ? i : i*stride[0] + offset; 931 } 932 933 /** 934 * @param i 935 * @param j 936 * @return the index on the data array corresponding to that location 937 */ 938 protected int get1DIndex(int i, int j) { 939 if (shape == null) { 940 throw new IllegalArgumentException("Cannot find an index from a null shape"); 941 } 942 if (shape.length != 2) { 943 logger.error("This dataset is not 2D but was addressed as such"); 944 throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such"); 945 } 946 if (i < 0) { 947 i += shape[0]; 948 } 949 if (i < 0 || i >= shape[0]) { 950 throwAIOOBException(i, shape[0], 0); 951 } 952 if (j < 0) { 953 j += shape[1]; 954 } 955 if (j < 0 || j >= shape[1]) { 956 throwAIOOBException(i, shape[1], 1); 957 } 958 return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset; 959 } 960 961 protected int get1DIndexFromShape(final int[] n) { 962 return get1DIndexFromShape(shape, n); 963 } 964 965 protected static int get1DIndexFromShape(final int[] shape, final int[] n) { 966 if (shape == null) { 967 throw new IllegalArgumentException("Cannot find an index from a null shape"); 968 } 969 final int rank = shape.length; 970 if (rank != n.length) { 971 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 972 logger.error(errMsg); 973 throw new IllegalArgumentException(errMsg); 974 } 975 int index = 0; 976 for (int i = 0; i < rank; i++) { 977 final int si = shape[i]; 978 int ni = n[i]; 979 if (ni < 0) { 980 ni += si; 981 } 982 if (ni < 0 || ni >= si) { 983 throwAIOOBException(ni, si, i); 984 } 985 index = index * si + ni; 986 } 987 988 return index; 989 } 990 991 private int get1DIndexFromStrides(final int[] n) { 992 return get1DIndexFromStrides(shape, stride, offset, n); 993 } 994 995 private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) { 996 if (shape == null) { 997 throw new IllegalArgumentException("Cannot find an index from a null shape"); 998 } 999 final int rank = shape.length; 1000 if (rank != n.length) { 1001 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 1002 logger.error(errMsg); 1003 throw new IllegalArgumentException(errMsg); 1004 } 1005 int index = offset; 1006 for (int i = 0; i < rank; i++) { 1007 final int st = stride[i]; 1008 if (st != 0) { // not broadcasted 1009 final int si = shape[i]; 1010 int ni = n[i]; 1011 if (ni < 0) { 1012 ni += si; 1013 } 1014 if (ni < 0 || ni >= si) { 1015 throwAIOOBException(ni, si, i); 1016 } 1017 index += st * ni; 1018 } 1019 } 1020 return index; 1021 } 1022 1023 @Override 1024 public int[] getNDPosition(final int n) { 1025 if (isIndexInRange(n)) { 1026 throw new IllegalArgumentException("Index provided " + n 1027 + "is larger then the size of the containing array"); 1028 } 1029 1030 return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n); 1031 } 1032 1033 private boolean isIndexInRange(final int n) { 1034 if (stride == null) { 1035 return n >= size; 1036 } 1037 return n >= getBufferLength(); 1038 } 1039 1040 /** 1041 * @return entire buffer length 1042 */ 1043 abstract protected int getBufferLength(); 1044 1045 private int[] getNDPositionFromStrides(int n) { 1046 n -= offset; 1047 int rank = shape.length; 1048 if (rank == 1) { 1049 return new int[] { n / stride[0] }; 1050 } 1051 1052 int[] output = new int[rank]; 1053 int i = 0; 1054 while (i != n) { // TODO find more efficient way than this exhaustive search 1055 int j = rank - 1; 1056 for (; j >= 0; j--) { 1057 output[j]++; 1058 i += stride[j]; 1059 if (output[j] >= shape[j]) { 1060 output[j] = 0; 1061 i -= shape[j] * stride[j]; 1062 } else { 1063 break; 1064 } 1065 } 1066 if (j == -1) { 1067 logger.error("Index was not found in this strided dataset"); 1068 throw new IllegalArgumentException("Index was not found in this strided dataset"); 1069 } 1070 } 1071 1072 return output; 1073 } 1074 1075 @Override 1076 public int checkAxis(int axis) { 1077 return ShapeUtils.checkAxis(shape.length, axis); 1078 } 1079 1080 @Deprecated 1081 protected static int checkAxis(int rank, int axis) { 1082 return ShapeUtils.checkAxis(rank, axis); 1083 } 1084 1085 protected static final char BLOCK_OPEN = '['; 1086 protected static final char BLOCK_CLOSE = ']'; 1087 1088 @Override 1089 public String toString() { 1090 final int rank = shape == null ? 0 : shape.length; 1091 final StringBuilder out = new StringBuilder(); 1092 1093 if (DTypeUtils.isDTypeElemental(getDType())) { 1094 out.append("Dataset "); 1095 } else { 1096 out.append("Compound dataset ("); 1097 out.append(getElementsPerItem()); 1098 out.append(") "); 1099 } 1100 1101 if (name != null && name.length() > 0) { 1102 out.append("'"); 1103 out.append(name); 1104 out.append("' has shape "); 1105 } else { 1106 out.append("shape is "); 1107 } 1108 1109 out.append(BLOCK_OPEN); 1110 if (rank > 0) { 1111 out.append(shape[0]); 1112 } 1113 for (int i = 1; i < rank; i++) { 1114 out.append(", " + shape[i]); 1115 } 1116 out.append(BLOCK_CLOSE); 1117 return out.toString(); 1118 } 1119 1120 @Override 1121 public String toString(boolean showData) { 1122 if (!showData) { 1123 return toString(); 1124 } 1125 1126 if (size == 0) { 1127 return "[]"; 1128 } 1129 1130 final int rank = shape == null ? 0 : shape.length; 1131 final StringBuilder out = new StringBuilder(); 1132 1133 if (rank > 0) { 1134 int[] pos = new int[rank]; 1135 final StringBuilder lead = new StringBuilder(); 1136 printBlocks(out, lead, 0, pos); 1137 } else { 1138 out.append(getString()); 1139 } 1140 return out.toString(); 1141 } 1142 1143 /** 1144 * Limit to strings output via the toString() method 1145 */ 1146 private static int maxStringLength = 120; 1147 1148 /** 1149 * Set maximum line length for toString() method 1150 * @param maxLineLength 1151 */ 1152 public static void setMaxLineLength(int maxLineLength) { 1153 maxStringLength = maxLineLength; 1154 } 1155 1156 /** 1157 * @return maximum line length for toString() method 1158 */ 1159 public static int getMaxLineLength() { 1160 return maxStringLength; 1161 } 1162 1163 /** 1164 * Limit to number of sub-blocks output via the toString() method 1165 */ 1166 private static final int MAX_SUBBLOCKS = 6; 1167 1168 private final static String SEPARATOR = ","; 1169 private final static String SPACE = " "; 1170 private final static String ELLIPSIS = "..."; 1171 private final static String NEWLINE = "\n"; 1172 1173 /** 1174 * Make a line of output for last dimension of dataset 1175 * 1176 * @param start 1177 * @return line 1178 */ 1179 private StringBuilder makeLine(final int end, final int[] start) { 1180 StringBuilder line = new StringBuilder(); 1181 final int[] pos; 1182 if (end >= start.length) { 1183 pos = Arrays.copyOf(start, end + 1); 1184 } else { 1185 pos = start; 1186 } 1187 pos[end] = 0; 1188 line.append(BLOCK_OPEN); 1189 line.append(getString(pos)); 1190 1191 final int length = shape[end]; 1192 1193 // trim elements printed if length exceed estimate of maximum elements 1194 int excess = length - maxStringLength / 3; // space + number + separator 1195 int midIndex = -1; 1196 if (excess > 0) { 1197 int index = (length - excess) / 2; 1198 for (int y = 1; y < index; y++) { 1199 line.append(SEPARATOR + SPACE); 1200 pos[end] = y; 1201 line.append(getString(pos)); 1202 } 1203 midIndex = line.length() + 2; 1204 index = (length + excess) / 2; 1205 for (int y = index; y < length; y++) { 1206 line.append(SEPARATOR + SPACE); 1207 pos[end] = y; 1208 line.append(getString(pos)); 1209 } 1210 } else { 1211 for (int y = 1; y < length; y++) { 1212 line.append(SEPARATOR + SPACE); 1213 pos[end] = y; 1214 line.append(getString(pos)); 1215 } 1216 } 1217 line.append(BLOCK_CLOSE); 1218 1219 // trim string down to limit 1220 int lineLength = line.length(); 1221 excess = lineLength - maxStringLength - ELLIPSIS.length() - 1; 1222 if (excess > 0) { 1223 int index = (lineLength - excess) / 2; 1224 if (midIndex > 0 && index > midIndex) { 1225 index = midIndex; 1226 } else { 1227 index = line.lastIndexOf(SEPARATOR, index) + 2; 1228 } 1229 StringBuilder out = new StringBuilder(line.subSequence(0, index)); 1230 out.append(ELLIPSIS + SEPARATOR); 1231 index = (lineLength + excess) / 2; 1232 if (midIndex > 0 && index <= midIndex) { 1233 index = midIndex - 1; 1234 } else { 1235 index = line.indexOf(SEPARATOR, index) + 1; 1236 } 1237 out.append(line.subSequence(index, lineLength)); 1238 return out; 1239 } else if (midIndex > 0) { // add ellipsis 1240 StringBuilder out = new StringBuilder(line.subSequence(0, midIndex)); 1241 out.append(ELLIPSIS + SEPARATOR + SPACE); 1242 out.append(line.subSequence(midIndex, lineLength)); 1243 return out; 1244 } 1245 1246 return line; 1247 } 1248 1249 /** 1250 * recursive method to print blocks 1251 */ 1252 private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) { 1253 if (out.length() > 0) { 1254 char last = out.charAt(out.length() - 1); 1255 if (last != BLOCK_OPEN) { 1256 out.append(lead); 1257 } 1258 } 1259 final int end = getRank() - 1; 1260 if (level != end) { 1261 out.append(BLOCK_OPEN); 1262 int length = shape[level]; 1263 1264 // first sub-block 1265 pos[level] = 0; 1266 StringBuilder newlead = new StringBuilder(lead); 1267 newlead.append(SPACE); 1268 printBlocks(out, newlead, level + 1, pos); 1269 if (length < 2) { // escape 1270 out.append(BLOCK_CLOSE); 1271 return; 1272 } 1273 1274 out.append(SEPARATOR + NEWLINE); 1275 for (int i = level + 1; i < end; i++) { 1276 out.append(NEWLINE); 1277 } 1278 1279 // middle sub-blocks 1280 if (length < MAX_SUBBLOCKS) { 1281 for (int x = 1; x < length - 1; x++) { 1282 pos[level] = x; 1283 printBlocks(out, newlead, level + 1, pos); 1284 if (end <= level + 1) { 1285 out.append(SEPARATOR + NEWLINE); 1286 } else { 1287 out.append(SEPARATOR + NEWLINE + NEWLINE); 1288 } 1289 } 1290 } else { 1291 final int excess = length - MAX_SUBBLOCKS; 1292 int xmax = (length - excess) / 2; 1293 for (int x = 1; x < xmax; x++) { 1294 pos[level] = x; 1295 printBlocks(out, newlead, level + 1, pos); 1296 if (end <= level + 1) { 1297 out.append(SEPARATOR + NEWLINE); 1298 } else { 1299 out.append(SEPARATOR + NEWLINE + NEWLINE); 1300 } 1301 } 1302 out.append(newlead); 1303 out.append(ELLIPSIS + SEPARATOR + NEWLINE); 1304 xmax = (length + excess) / 2; 1305 for (int x = xmax; x < length - 1; x++) { 1306 pos[level] = x; 1307 printBlocks(out, newlead, level + 1, pos); 1308 if (end <= level + 1) { 1309 out.append(SEPARATOR + NEWLINE); 1310 } else { 1311 out.append(SEPARATOR + NEWLINE + NEWLINE); 1312 } 1313 } 1314 } 1315 1316 // last sub-block 1317 pos[level] = length - 1; 1318 printBlocks(out, newlead, level + 1, pos); 1319 out.append(BLOCK_CLOSE); 1320 } else { 1321 out.append(makeLine(end, pos)); 1322 } 1323 } 1324 1325 @Override 1326 public Dataset squeezeEnds() { 1327 return squeeze(true); 1328 } 1329 1330 @Override 1331 public Dataset squeeze() { 1332 return squeeze(false); 1333 } 1334 1335 @Override 1336 public Dataset squeeze(boolean onlyFromEnds) { 1337 final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds); 1338 final int[] oshape = shape; 1339 if (stride == null) { 1340 shape = tshape; 1341 } else { 1342 int rank = shape.length; 1343 int trank = tshape.length; 1344 if (trank < rank) { 1345 int[] tstride = new int[tshape.length]; 1346 if (onlyFromEnds) { 1347 for (int i = 0; i < rank; i++) { 1348 if (shape[i] != 1) { 1349 for (int k = 0; k < trank; k++) { 1350 tstride[k] = stride[i++]; 1351 } 1352 break; 1353 } 1354 } 1355 } else { 1356 int t = 0; 1357 for (int i = 0; i < rank; i++) { 1358 if (shape[i] != 1) { 1359 tstride[t++] = stride[i]; 1360 } 1361 } 1362 } 1363 shape = tshape; 1364 stride = tstride; 1365 } 1366 } 1367 1368 setDirty(); 1369 reshapeMetadata(oshape, shape); 1370 return this; 1371 } 1372 1373 @Override 1374 public boolean isCompatibleWith(final ILazyDataset g) { 1375 return ShapeUtils.areShapesCompatible(shape, g.getShape()); 1376 } 1377 1378 @Override 1379 public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException { 1380 ShapeUtils.checkCompatibility(this, g); 1381 } 1382 1383 @Override 1384 public Dataset reshape(final int... shape) { 1385 Dataset a; 1386 try { 1387 a = getView(true); 1388 a.setShape(shape); 1389 } catch (IllegalArgumentException e) { 1390 a = clone(); 1391 a.setShape(shape); 1392 } 1393 return a; 1394 } 1395 1396 /** 1397 * @param start 1398 * @param stop 1399 * @param step 1400 * @return number of steps to take 1401 */ 1402 protected static int calcSteps(final double start, final double stop, final double step) { 1403 return Math.max(0, (int) Math.ceil((stop - start) / step)); 1404 } 1405 1406 @Override 1407 public boolean isComplex() { 1408 int type = getDType(); 1409 return type == COMPLEX64 || type == COMPLEX128; 1410 } 1411 1412 @Override 1413 public Dataset getRealPart() { 1414 return this; 1415 } 1416 1417 @Override 1418 public Dataset getRealView() { 1419 return getView(true); 1420 } 1421 1422 @Override 1423 public Dataset getSlice(final int[] start, final int[] stop, final int[] step) { 1424 return getSlice(new SliceND(shape, start, stop, step)); 1425 } 1426 1427 @Override 1428 public Dataset getSlice(Slice... slice) { 1429 return getSlice(new SliceND(shape, slice)); 1430 } 1431 1432 @Override 1433 public Dataset getSlice(IMonitor monitor, Slice... slice) { 1434 return getSlice(slice); 1435 } 1436 1437 @Override 1438 public Dataset getSlice(IMonitor monitor, SliceND slice) { 1439 return getSlice(slice); 1440 } 1441 1442 @Override 1443 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) { 1444 return getSlice(start, stop, step); 1445 } 1446 1447 /** 1448 * Get a slice of the dataset. The returned dataset is a copied selection of items 1449 * @param slice 1450 * @return The dataset of the sliced data 1451 */ 1452 @Override 1453 public Dataset getSlice(final SliceND slice) { 1454 SliceIterator it = (SliceIterator) getSliceIterator(slice); 1455 AbstractDataset s = getSlice(it); 1456 s.metadata = copyMetadata(); 1457 s.setDirty(); 1458 s.sliceMetadata(true, slice); 1459 return s; 1460 } 1461 1462 /** 1463 * Get a slice of the dataset. The returned dataset is a copied selection of items 1464 * 1465 * @param iterator Slice iterator 1466 * @return The dataset of the sliced data 1467 */ 1468 abstract public AbstractDataset getSlice(final SliceIterator iterator); 1469 1470 @SuppressWarnings("deprecation") 1471 @Override 1472 public Dataset setSlice(final Object obj, final SliceND slice) { 1473 Dataset ds; 1474 if (obj instanceof Dataset) { 1475 ds = (Dataset) obj; 1476 } else if (obj instanceof IDataset) { 1477 ds = DatasetUtils.convertToDataset((IDataset) obj); 1478 } else { 1479 int dtype = getDType(); 1480 if (dtype != Dataset.BOOL) { 1481 dtype = DTypeUtils.getLargestDType(dtype); 1482 } 1483 ds = DatasetFactory.createFromObject(getElementsPerItem(), dtype, obj); 1484 } 1485 1486 return setSlicedView(getSliceView(slice), ds); 1487 } 1488 1489 @Override 1490 public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) { 1491 return setSlice(obj, new SliceND(shape, start, stop, step)); 1492 } 1493 1494 /** 1495 * Set a view of current dataset to given dataset with broadcasting 1496 * @param view 1497 * @param d 1498 * @return this dataset 1499 */ 1500 abstract Dataset setSlicedView(Dataset view, Dataset d); 1501 1502 @Override 1503 public Dataset setSlice(Object obj, Slice... slice) { 1504 if (slice == null || slice.length == 0) { 1505 return setSlice(obj, new SliceND(shape)); 1506 } 1507 return setSlice(obj, new SliceND(shape, slice)); 1508 } 1509 1510 @Override 1511 public boolean all() { 1512 return Comparisons.allTrue(this); 1513 } 1514 1515 @Override 1516 public BooleanDataset all(final int axis) { 1517 return Comparisons.allTrue(this, axis); 1518 } 1519 1520 @Override 1521 public boolean any() { 1522 return Comparisons.anyTrue(this); 1523 } 1524 1525 @Override 1526 public BooleanDataset any(final int axis) { 1527 return Comparisons.anyTrue(this, axis); 1528 } 1529 1530 @Override 1531 public Dataset ifloorDivide(final Object o) { 1532 return idivide(o).ifloor(); 1533 } 1534 1535 @Override 1536 public double residual(final Object o) { 1537 return residual(o, null, false); 1538 } 1539 1540 @Override 1541 public double residual(final Object o, boolean ignoreNaNs) { 1542 return residual(o, null, ignoreNaNs); 1543 } 1544 1545 /** 1546 * @since 2.0 1547 */ 1548 @SuppressWarnings("unchecked") 1549 protected StatisticsMetadata<Number> getStats() { 1550 StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class); 1551 if (md == null || md.isDirty()) { 1552 md = new StatisticsMetadataImpl<Number>(); 1553 md.initialize(this); 1554 setMetadata(md); 1555 } 1556 return md; 1557 } 1558 1559 /** 1560 * @since 2.0 1561 */ 1562 @SuppressWarnings("unchecked") 1563 protected StatisticsMetadata<String> getStringStats() { 1564 StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class); 1565 if (md == null || md.isDirty()) { 1566 md = new StatisticsMetadataImpl<String>(); 1567 md.initialize(this); 1568 setMetadata(md); 1569 } 1570 return md; 1571 } 1572 1573 @Override 1574 public Number max(boolean... ignoreInvalids) { 1575 return getStats().getMaximum(ignoreInvalids); 1576 } 1577 1578 @Override 1579 public Dataset max(int axis, boolean... ignoreInvalids) { 1580 return getStats().getMaximum(axis, ignoreInvalids); 1581 } 1582 1583 @Override 1584 public Number min(boolean... ignoreInvalids) { 1585 return getStats().getMinimum(ignoreInvalids); 1586 } 1587 1588 @Override 1589 public Dataset min(int axis, boolean... ignoreInvalids) { 1590 return getStats().getMinimum(axis, ignoreInvalids); 1591 } 1592 1593 @Override 1594 public int argMax(boolean... ignoreInvalids) { 1595 return getFlat1DIndex(maxPos(ignoreInvalids)); 1596 } 1597 1598 /** 1599 * @since 2.0 1600 */ 1601 @Override 1602 public IntegerDataset argMax(int axis, boolean... ignoreInvalids) { 1603 return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids); 1604 } 1605 1606 @Override 1607 public int argMin(boolean... ignoreInvalids) { 1608 return getFlat1DIndex(minPos(ignoreInvalids)); 1609 } 1610 1611 /** 1612 * @since 2.0 1613 */ 1614 @Override 1615 public IntegerDataset argMin(int axis, boolean... ignoreInvalids) { 1616 return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids); 1617 } 1618 1619 @Override 1620 public Number peakToPeak(boolean... ignoreInvalids) { 1621 return DTypeUtils.fromDoubleToBiggestNumber(max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue(), getDType()); 1622 } 1623 1624 @Override 1625 public Dataset peakToPeak(int axis, boolean... ignoreInvalids) { 1626 return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids)); 1627 } 1628 1629 1630 @Override 1631 public long count(boolean... ignoreInvalids) { 1632 return getStats().getCount(ignoreInvalids); 1633 } 1634 1635 @Override 1636 public Dataset count(int axis, boolean... ignoreInvalids) { 1637 return getStats().getCount(axis, ignoreInvalids); 1638 } 1639 1640 @Override 1641 public Object sum(boolean... ignoreInvalids) { 1642 return getStats().getSum(ignoreInvalids); 1643 } 1644 1645 @Override 1646 public Dataset sum(int axis, boolean... ignoreInvalids) { 1647 return getStats().getSum(axis, ignoreInvalids); 1648 } 1649 1650 @Override 1651 public Object product(boolean... ignoreInvalids) { 1652 return Stats.product(this, ignoreInvalids); 1653 } 1654 1655 @Override 1656 public Dataset product(int axis, boolean... ignoreInvalids) { 1657 return Stats.product(this, axis, ignoreInvalids); 1658 } 1659 1660 @Override 1661 public Object mean(boolean... ignoreInvalids) { 1662 return getStats().getMean(ignoreInvalids); 1663 } 1664 1665 @Override 1666 public Dataset mean(int axis, boolean... ignoreInvalids) { 1667 return getStats().getMean(axis, ignoreInvalids); 1668 } 1669 1670 @Override 1671 public double variance() { 1672 return variance(false); 1673 } 1674 1675 @Override 1676 public double variance(boolean isWholePopulation, boolean... ignoreInvalids) { 1677 return getStats().getVariance(isWholePopulation, ignoreInvalids); 1678 } 1679 1680 @Override 1681 public Dataset variance(int axis) { 1682 return getStats().getVariance(axis, false); 1683 } 1684 1685 @Override 1686 public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1687 return getStats().getVariance(axis, isWholePopulation, ignoreInvalids); 1688 } 1689 1690 @Override 1691 public double stdDeviation() { 1692 return Math.sqrt(variance()); 1693 } 1694 1695 @Override 1696 public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) { 1697 return Math.sqrt(variance(isWholePopulation, ignoreInvalids)); 1698 } 1699 1700 @Override 1701 public Dataset stdDeviation(int axis) { 1702 return Maths.sqrt(variance(axis, false)); 1703 } 1704 1705 @Override 1706 public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1707 return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids)); 1708 } 1709 1710 @Override 1711 public double rootMeanSquare(boolean... ignoreInvalids) { 1712 StatisticsMetadata<Number> stats = getStats(); 1713 final double mean = stats.getMean(ignoreInvalids).doubleValue(); 1714 final double var = stats.getVariance(true, ignoreInvalids); 1715 return Math.sqrt(var + mean * mean); 1716 } 1717 1718 @Override 1719 public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) { 1720 StatisticsMetadata<Number> stats = getStats(); 1721 Dataset v = stats.getVariance(axis, true, ignoreInvalids); 1722 Dataset m = stats.getMean(axis, ignoreInvalids); 1723 Dataset result = Maths.multiply(m, m); 1724 return Maths.sqrt(result.iadd(v)); 1725 } 1726 1727 /** 1728 * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards. 1729 * 1730 * @param dindex 1731 * @param sindex 1732 * @param src 1733 * is the source data buffer 1734 */ 1735 protected abstract void setItemDirect(final int dindex, final int sindex, final Object src); 1736 1737 /** 1738 * @return error broadcasted to current shape 1739 */ 1740 private Dataset getBroadcastedInternalError() { 1741 if (shape == null) { 1742 throw new IllegalArgumentException("Cannot get error for null dataset"); 1743 } 1744 ILazyDataset led = super.getErrors(); 1745 if (led == null) 1746 return null; 1747 1748 Dataset ed = null; 1749 try { 1750 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1751 } catch (DatasetException e) { 1752 logger.error("Could not get data from lazy dataset", e); 1753 } 1754 if (led != ed) { 1755 setErrors(ed); // set back 1756 } 1757 1758 return ed.getBroadcastView(shape); 1759 } 1760 1761 @Override 1762 public Dataset getErrors() { 1763 Dataset ed = getBroadcastedInternalError(); 1764 if (ed == null) 1765 return null; 1766 1767 return ed; 1768 } 1769 1770 @Override 1771 public double getError() { 1772 Dataset ed = getBroadcastedInternalError(); 1773 if (ed == null) 1774 return 0; 1775 1776 return ed.getDouble(); 1777 } 1778 1779 @Override 1780 public double getError(final int i) { 1781 Dataset ed = getBroadcastedInternalError(); 1782 if (ed == null) 1783 return 0; 1784 1785 return ed.getDouble(i); 1786 } 1787 1788 @Override 1789 public double getError(final int i, final int j) { 1790 Dataset ed = getBroadcastedInternalError(); 1791 if (ed == null) 1792 return 0; 1793 1794 return ed.getDouble(i, j); 1795 } 1796 1797 @Override 1798 public double getError(int... pos) { 1799 Dataset ed = getBroadcastedInternalError(); 1800 if (ed == null) 1801 return 0; 1802 1803 return ed.getDouble(pos); 1804 } 1805 1806 @Override 1807 public double[] getErrorArray(final int i) { 1808 Dataset ed = getBroadcastedInternalError(); 1809 if (ed == null) 1810 return null; 1811 1812 return new double[] {getError(i)}; 1813 } 1814 1815 @Override 1816 public double[] getErrorArray(final int i, final int j) { 1817 Dataset ed = getBroadcastedInternalError(); 1818 if (ed == null) 1819 return null; 1820 1821 return new double[] {getError(i, j)}; 1822 } 1823 1824 @Override 1825 public double[] getErrorArray(int... pos) { 1826 Dataset ed = getBroadcastedInternalError(); 1827 if (ed == null) 1828 return null; 1829 1830 return new double[] {getError(pos)}; 1831 } 1832 1833 protected Dataset getInternalSquaredError() { 1834 Dataset sed = getErrorBuffer().getBroadcastView(shape); 1835 return sed; 1836 } 1837 1838 @Override 1839 public Dataset getErrorBuffer() { 1840 ErrorMetadata emd = getErrorMetadata(); 1841 if (emd == null) 1842 return null; 1843 1844 if (!(emd instanceof ErrorMetadataImpl)) { 1845 ILazyDataset led = emd.getError(); 1846 Dataset ed; 1847 try { 1848 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1849 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1850 setMetadata(emd); 1851 emd.setError(ed); 1852 } catch (MetadataException me) { 1853 logger.error("Could not create metadata", me); 1854 } catch (DatasetException e) { 1855 logger.error("Could not get data from lazy dataset", e); 1856 } 1857 } 1858 1859 return ((ErrorMetadataImpl) emd).getSquaredError(); 1860 } 1861 1862 /** 1863 * Set a copy of the buffer that backs the (squared) error data 1864 * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset 1865 */ 1866 @Override 1867 public void setErrorBuffer(Serializable buffer) { 1868 if (shape == null) { 1869 throw new IllegalArgumentException("Cannot set error buffer for null dataset"); 1870 } 1871 if (buffer == null) { 1872 clearMetadata(ErrorMetadata.class); 1873 return; 1874 } 1875 1876 IDataset d = (IDataset) createFromSerializable(buffer, false); 1877 ErrorMetadata emd = getErrorMetadata(); 1878 if (!(emd instanceof ErrorMetadataImpl)) { 1879 try { 1880 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1881 setMetadata(emd); 1882 } catch (MetadataException me) { 1883 logger.error("Could not create metadata", me); 1884 } 1885 } 1886 ((ErrorMetadataImpl) emd).setSquaredError(d); 1887 } 1888}