001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.reflect.Array; 017import java.text.Format; 018import java.util.Arrays; 019import java.util.List; 020 021import org.eclipse.january.DatasetException; 022import org.eclipse.january.IMonitor; 023import org.eclipse.january.MetadataException; 024import org.eclipse.january.metadata.ErrorMetadata; 025import org.eclipse.january.metadata.MetadataFactory; 026import org.eclipse.january.metadata.StatisticsMetadata; 027import org.eclipse.january.metadata.internal.ErrorMetadataImpl; 028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl; 029 030/** 031 * Generic container class for data 032 * <p> 033 * Each subclass has an array of primitive types, elements of this array are grouped or 034 * compounded to make items 035 * <p> 036 * Data items can be boolean, integer, float, complex float, vector float, etc 037 */ 038public abstract class AbstractDataset extends LazyDatasetBase implements Dataset { 039 // pin UID to base class 040 private static final long serialVersionUID = Dataset.serialVersionUID; 041 042 protected int size; // number of items 043 044 transient protected AbstractDataset base; // is null when not a view 045 protected int[] stride; // can be null for row-major, contiguous datasets 046 protected int offset; 047 048 /** 049 * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as 050 * wanted 051 */ 052 protected Serializable odata = null; 053 054 /** 055 * Set aliased data as base data 056 */ 057 abstract protected void setData(); 058 059 /** 060 * Constructor required for serialisation. 061 */ 062 public AbstractDataset() { 063 } 064 065 @Override 066 public synchronized Dataset synchronizedCopy() { 067 return clone(); 068 } 069 070 @Override 071 public int hashCode() { 072 return getStats().getHash(shape); 073 } 074 075 @Override 076 abstract public AbstractDataset clone(); 077 078 protected Format stringFormat = null; 079 080 @Override 081 public void setStringFormat(Format format) { 082 stringFormat = format; 083 } 084 085 @Override 086 public Dataset copy(final int dtype) { 087 if (getDType() == dtype) { 088 return clone(); 089 } 090 return DatasetUtils.copy(this, dtype); 091 } 092 093 @Override 094 public <T extends Dataset> T copy(Class<T> clazz) { 095 return DatasetUtils.copy(clazz, this); 096 } 097 098 @Override 099 public Dataset cast(final int dtype) { 100 if (getDType() == dtype) { 101 return this; 102 } 103 return DatasetUtils.cast(this, dtype); 104 } 105 106 @Override 107 public <T extends Dataset> T cast(Class<T> clazz) { 108 return DatasetUtils.cast(clazz, this); 109 } 110 111 @Override 112 public Dataset cast(final boolean repeat, final int dtype, final int isize) { 113 if (getDType() == dtype && getElementsPerItem() == isize) { 114 return this; 115 } 116 return DatasetUtils.cast(this, repeat, dtype, isize); 117 } 118 119 @Override 120 abstract public AbstractDataset getView(boolean deepCopyMetadata); 121 122 /** 123 * Copy fields from original to view 124 * @param orig 125 * @param view 126 * @param clone if true, then clone everything but bulk data 127 * @param cloneMetadata if true, clone metadata 128 */ 129 protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) { 130 view.name = orig.getName(); 131 view.size = orig.getSize(); 132 view.odata = orig.getBuffer(); 133 view.offset = orig.getOffset(); 134 view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null; 135 136 if (clone) { 137 view.shape = orig.getShape(); 138 view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ? 139 ((AbstractDataset) orig).stride.clone() : null; 140 } else { 141 view.shape = orig.getShapeRef(); 142 view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null; 143 } 144 145 view.metadata = getMetadataMap(orig, cloneMetadata); 146 int odtype = orig.getDType(); 147 int vdtype = view.getDType(); 148 if (odtype != vdtype) { 149 view.setDirty(); 150 } 151 } 152 153 @Override 154 public IntegerDataset getIndices() { 155 final IntegerDataset ret = DatasetUtils.indices(shape); 156 if (getName() != null) { 157 ret.setName("Indices of " + getName()); 158 } 159 return ret; 160 } 161 162 @Override 163 public Dataset getTransposedView(int... axes) { 164 axes = checkPermutatedAxes(shape, axes); 165 166 AbstractDataset t = getView(true); 167 if (axes == null || getRank() == 1) 168 return t; 169 170 int rank = shape.length; 171 int[] tstride = new int[rank]; 172 int[] toffset = new int[1]; 173 int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset); 174 int[] nstride = new int[rank]; 175 for (int i = 0; i < rank; i++) { 176 final int ax = axes[i]; 177 nstride[i] = tstride[ax]; 178 nshape[i] = shape[ax]; 179 } 180 t.shape = nshape; 181 t.stride = nstride; 182 t.offset = toffset[0]; 183 t.base = this; 184 t.setDirty(); 185 t.transposeMetadata(axes); 186 return t; 187 } 188 189 @Override 190 public Dataset transpose(int... axes) { 191 Dataset t = getTransposedView(axes); 192 return t == null ? clone() : t.clone(); 193 } 194 195 @Override 196 public Dataset swapAxes(int axis1, int axis2) { 197 int rank = shape.length; 198 axis1 = ShapeUtils.checkAxis(rank, axis1); 199 axis2 = ShapeUtils.checkAxis(rank, axis2); 200 201 if (rank == 1 || axis1 == axis2) { 202 return this; 203 } 204 205 int[] axes = new int[rank]; 206 for (int i = 0; i < rank; i++) { 207 axes[i] = i; 208 } 209 210 axes[axis1] = axis2; 211 axes[axis2] = axis1; 212 return getTransposedView(axes); 213 } 214 215 boolean isContiguous() { 216 if (stride == null) 217 return true; 218 219 if (offset != 0) 220 return false; 221 222 int s = getElementsPerItem(); 223 for (int j = getRank() - 1; j >= 0; j--) { 224 if (stride[j] != s) { 225 return false; 226 } 227 s *= shape[j]; 228 } 229 230 return true; 231 } 232 233 @Override 234 public Dataset flatten() { 235 if (!isContiguous()) { // need to make a copy if not contiguous 236 return clone().flatten(); 237 } 238 return reshape(size); 239 } 240 241 /** 242 * Fill dataset from object at depth dimension 243 * @param obj 244 * @param depth 245 * @param pos position 246 */ 247 protected void fillData(Object obj, final int depth, final int[] pos) { 248 if (obj == null) { 249 int dtype = getDType(); 250 if (dtype == FLOAT32) 251 set(Float.NaN, pos); 252 else if (dtype == FLOAT64) 253 set(Double.NaN, pos); 254 return; 255 } 256 257 Class<?> clazz = obj.getClass(); 258 if (obj instanceof List<?>) { 259 List<?> jl = (List<?>) obj; 260 int l = jl.size(); 261 for (int i = 0; i < l; i++) { 262 Object lo = jl.get(i); 263 fillData(lo, depth + 1, pos); 264 pos[depth]++; 265 } 266 pos[depth] = 0; 267 } else if (clazz.isArray()) { 268 int l = Array.getLength(obj); 269 if (clazz.equals(odata.getClass())) { 270 System.arraycopy(obj, 0, odata, get1DIndex(pos), l); 271 } else if (clazz.getComponentType().isPrimitive()) { 272 for (int i = 0; i < l; i++) { 273 set(Array.get(obj, i), pos); 274 pos[depth]++; 275 } 276 pos[depth] = 0; 277 } else { 278 for (int i = 0; i < l; i++) { 279 fillData(Array.get(obj, i), depth + 1, pos); 280 pos[depth]++; 281 } 282 pos[depth] = 0; 283 } 284 } else if (obj instanceof IDataset) { 285 boolean[] a = new boolean[shape.length]; 286 for (int i = depth; i < a.length; i++) 287 a[i] = true; 288 setSlice(obj, getSliceIteratorFromAxes(pos, a)); 289 } else { 290 set(obj, pos); 291 } 292 } 293 294 @Override 295 public IndexIterator getIterator(final boolean withPosition) { 296 if (stride != null) { 297 return base.getSize() == 1 ? (withPosition ? new PositionIterator(offset, shape) : 298 new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset); 299 } 300 if (shape == null) { 301 return new NullIterator(shape, shape); 302 } 303 304 return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size); 305 } 306 307 @Override 308 public IndexIterator getIterator() { 309 return getIterator(false); 310 } 311 312 @Override 313 public PositionIterator getPositionIterator(final int... axes) { 314 return new PositionIterator(shape, axes); 315 } 316 317 @Override 318 public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) { 319 return getSliceIterator(new SliceND(shape, start, stop, step)); 320 } 321 322 /** 323 * @param slice 324 * @return an slice iterator that operates like an IndexIterator 325 */ 326 public IndexIterator getSliceIterator(SliceND slice) { 327 if (ShapeUtils.calcLongSize(slice.getShape()) == 0) { 328 return new NullIterator(shape, slice.getShape()); 329 } 330 if (stride != null) { 331 return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice); 332 } 333 return new SliceIterator(shape, size, slice); 334 } 335 336 @Override 337 public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) { 338 int rank = shape.length; 339 int[] start; 340 int[] stop = new int[rank]; 341 int[] step = new int[rank]; 342 343 if (pos == null) { 344 start = new int[rank]; 345 } else if (pos.length == rank) { 346 start = pos.clone(); 347 } else { 348 throw new IllegalArgumentException("pos array length is not equal to rank of dataset"); 349 } 350 if (axes == null) { 351 axes = new boolean[rank]; 352 Arrays.fill(axes, true); 353 } else if (axes.length != rank) { 354 throw new IllegalArgumentException("axes array length is not equal to rank of dataset"); 355 } 356 357 for (int i = 0; i < rank; i++) { 358 if (axes[i]) { 359 stop[i] = shape[i]; 360 } else { 361 stop[i] = start[i] + 1; 362 } 363 step[i] = 1; 364 } 365 return (SliceIterator) getSliceIterator(start, stop, step); 366 } 367 368 @Override 369 public BooleanIterator getBooleanIterator(Dataset choice) { 370 return getBooleanIterator(choice, true); 371 } 372 373 @Override 374 public BooleanIterator getBooleanIterator(Dataset choice, boolean value) { 375 return BooleanIterator.createIterator(value, this, choice, this); 376 } 377 378 @Override 379 public Dataset getByBoolean(Dataset selection) { 380 checkCompatibility(selection); 381 382 final int length = ((Number) selection.sum()).intValue(); 383 final int is = getElementsPerItem(); 384 Dataset r = DatasetFactory.zeros(is, getClass(), length); 385 BooleanIterator biter = getBooleanIterator(selection); 386 387 int i = 0; 388 while (biter.hasNext()) { 389 r.setObjectAbs(i, getObjectAbs(biter.index)); 390 i += is; 391 } 392 return r; 393 } 394 395 @Override 396 public Dataset getBy1DIndex(IntegerDataset index) { 397 final int is = getElementsPerItem(); 398 final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShape()); 399 final IntegerIterator iter = new IntegerIterator(index, size, is); 400 401 int i = 0; 402 while (iter.hasNext()) { 403 r.setObjectAbs(i, getObjectAbs(iter.index)); 404 i += is; 405 } 406 return r; 407 } 408 409 @Override 410 public Dataset getByIndexes(final Object... indexes) { 411 final IntegersIterator iter = new IntegersIterator(shape, indexes); 412 final int is = getElementsPerItem(); 413 final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape()); 414 415 final int[] pos = iter.getPos(); 416 int i = 0; 417 while (iter.hasNext()) { 418 r.setObjectAbs(i, getObject(pos)); 419 i += is; 420 } 421 return r; 422 } 423 424 @Override 425 public Class<?> getElementClass() { 426 return DTypeUtils.getElementClass(getDType()); 427 } 428 429 @Override 430 public boolean hasFloatingPointElements() { 431 Class<?> cls = getElementClass(); 432 return cls == Float.class || cls == Double.class; 433 } 434 435 @Override 436 public int getElementsPerItem() { 437 return DTypeUtils.getElementsPerItem(getDType()); 438 } 439 440 @Override 441 public int getItemBytes() { 442 return DTypeUtils.getItemBytes(getDType(), getElementsPerItem()); 443 } 444 445 @Override 446 public int getSize() { 447 return size; 448 } 449 450 @Override 451 public int[] getShape() { 452 // make a copy of the dimensions data, and put that out 453 if (shape == null) { 454 logger.warn("Shape is null!!!"); 455 return new int[] {}; 456 } 457 return shape.clone(); 458 } 459 460 @Override 461 public int getRank() { 462 return shape == null ? 0 : shape.length; 463 } 464 465 @Override 466 public int getNbytes() { 467 return getSize() * getItemBytes(); 468 } 469 470 /** 471 * Check for -1 placeholder in shape and replace if necessary 472 * @param shape 473 * @param size 474 */ 475 private void checkShape(int[] shape, int size) { 476 int rank = shape.length; 477 int found = -1; 478 int nsize = 1; 479 for (int i = 0; i < rank; i++) { 480 int d = shape[i]; 481 if (d == -1) { 482 if (found == -1) { 483 found = i; 484 } else { 485 logger.error("Can only have one -1 placeholder in shape"); 486 throw new IllegalArgumentException("Can only have one -1 placeholder in shape"); 487 } 488 } else { 489 nsize *= d; 490 } 491 } 492 if (found >= 0) { 493 shape[found] = size/nsize; 494 } else if (nsize != size && !(rank == 0 && size == 0)) { 495 logger.error("New shape is not same size as old shape"); 496 throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape)); 497 } 498 } 499 500 @Override 501 public void setShape(final int... shape) { 502 int[] nshape = shape.clone(); 503 checkShape(nshape, size); 504 if (Arrays.equals(this.shape, nshape)) { 505 return; 506 } 507 508 if (stride != null) { 509 // the only compatible shapes are ones where new dimensions are factors of old dimensions 510 // or are combined adjacent old dimensions 511 int[] oshape = this.shape; 512 int orank = oshape.length; 513 int nrank = nshape.length; 514 int diff = nrank - orank; 515 int[] nstride = new int[nrank]; 516 boolean ones = true; 517 // work forwards for broadcasting cases 518 for (int i = 0, j = 0; i < orank || j < nrank;) { 519 if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) { 520 nstride[j++] = stride[i++]; 521 } else if (j < nrank && nshape[j] == 1) { 522 nstride[j++] = 0; 523 } else if (i < orank && oshape[i] == 1) { 524 i++; 525 } else { 526 if (j < nrank) 527 j++; 528 if (i < orank) 529 i++; 530 ones = false; 531 } 532 } 533 if (!ones) { // not just ones differ in shapes 534 int[] ostride = stride; 535 int ob = 0; 536 int oe = 1; 537 int nb = 0; 538 int ne = 1; 539 while (ob < orank && nb < nrank) { 540 int ol = oshape[ob]; 541 int nl = nshape[nb]; 542 543 if (nl < ol) { // find group of shape dimensions that form common size 544 do { // case where new shape spreads single dimension over several dimensions 545 if (ne == nrank) { 546 break; 547 } 548 nl *= nshape[ne++]; 549 } while (nl < ol); 550 if (nl != ol) { 551 logger.error("Subshape is incompatible with single dimension"); 552 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 553 } 554 int on = ne - 1; 555 while (nshape[on] == 1) { 556 on--; 557 } 558 559 nstride[on] = ostride[ob]; 560 for (int n = on - 1; n >= nb; n--) { 561 if (nshape[n] == 1) 562 continue; 563 564 nstride[n] = nshape[on] * nstride[on]; 565 on = n; 566 } 567 } else if (ol < nl) { 568 do { // case where new shape combines several dimensions into one dimension 569 if (oe == orank) { 570 break; 571 } 572 ol *= oshape[oe++]; 573 } while (ol < nl); 574 if (nl != ol) { 575 logger.error("Single dimension is incompatible with subshape"); 576 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 577 } 578 579 int oo = oe - 1; 580 while (oshape[oo] == 1) { 581 oo--; 582 } 583 int os = ostride[oo]; 584 for (int o = oo - 1; o >= ob; o--) { 585 if (oshape[o] == 1) 586 continue; 587 if (ostride[o] != oshape[oo] * ostride[oo]) { 588 logger.error("Subshape cannot be a non-contiguous view"); 589 throw new IllegalArgumentException("Subshape cannot be a non-contiguous view"); 590 } 591 oo = o; 592 } 593 nstride[nb] = os; 594 } else { 595 nstride[nb] = ostride[ob]; 596 } 597 598 ob = oe++; 599 nb = ne++; 600 } 601 } 602 603 stride = nstride; 604 } 605 606 setDirty(); 607 if (this.shape != null && metadata != null) { 608 reshapeMetadata(this.shape, nshape); 609 } 610 this.shape = nshape; 611 } 612 613 @Override 614 public int[] getShapeRef() { 615 return shape; 616 } 617 618 @Override 619 public int getOffset() { 620 return offset; 621 } 622 623 @Override 624 public int[] getStrides() { 625 return stride; 626 } 627 628 @Override 629 public Serializable getBuffer() { 630 return odata; 631 } 632 633 @Override 634 public void overrideInternal(Serializable buffer, int... shape) { 635 if (buffer != null) { 636 odata = buffer; 637 setData(); 638 setDirty(); 639 } 640 641 if (shape != null) { 642 this.shape = shape.clone(); 643 size = ShapeUtils.calcSize(this.shape); 644 } 645 } 646 647 /** 648 * Create a stride array from dataset 649 * @param a dataset 650 * @param offset output offset 651 * @return new strides 652 */ 653 public static int[] createStrides(Dataset a, final int[] offset) { 654 return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset); 655 } 656 657 /** 658 * Create a stride array from dataset 659 * @param isize 660 * @param shape 661 * @param oStride original stride 662 * @param oOffset original offset (only used if there is an original stride) 663 * @param offset output offset 664 * @return new strides 665 */ 666 public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) { 667 int rank = shape.length; 668 final int[] stride; 669 if (oStride == null) { 670 offset[0] = 0; 671 stride = new int[rank]; 672 int s = isize; 673 for (int j = rank - 1; j >= 0; j--) { 674 stride[j] = s; 675 s *= shape[j]; 676 } 677 } else { 678 offset[0] = oOffset; 679 stride = oStride.clone(); 680 } 681 return stride; 682 } 683 684 /** 685 * Create a stride array from slice information and a dataset 686 * @param slice 687 * @param a dataset 688 * @param stride output stride 689 * @param offset output offset 690 * @return new shape 691 */ 692 public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) { 693 return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset); 694 } 695 696 /** 697 * Create a stride array from slice and dataset information 698 * @param slice 699 * @param isize 700 * @param shape 701 * @param oStride original stride 702 * @param oOffset original offset (only used if there is an original stride) 703 * @param stride output stride 704 * @param offset output offset 705 * @return new shape 706 */ 707 public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) { 708 int[] lstart = slice.getStart(); 709 int[] lstep = slice.getStep(); 710 int[] newShape = slice.getShape(); 711 int rank = shape.length; 712 713 if (oStride == null) { 714 int s = isize; 715 offset[0] = 0; 716 for (int j = rank - 1; j >= 0; j--) { 717 stride[j] = s * lstep[j]; 718 offset[0] += s * lstart[j]; 719 s *= shape[j]; 720 } 721 } else { 722 offset[0] = oOffset; 723 for (int j = 0; j < rank; j++) { 724 int s = oStride[j]; 725 stride[j] = lstep[j] * s; 726 offset[0] += lstart[j] * s; 727 } 728 } 729 730 return newShape; 731 } 732 733 @Override 734 public Dataset getBroadcastView(int... broadcastShape) { 735 AbstractDataset view = getView(true); 736 737 if (!Arrays.equals(shape, broadcastShape)) { 738 List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape); 739 view.setShape(nShapes.get(0)); 740 view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape); 741 view.base = this; 742 view.shape = broadcastShape.clone(); 743 view.size = ShapeUtils.calcSize(broadcastShape); 744 if (view.name == null || view.name.isEmpty()) { 745 view.name = "Broadcast from " + Arrays.toString(shape); 746 } else { 747 view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape); 748 } 749 } 750 return view; 751 } 752 753 @Override 754 public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) { 755 return getSliceView(new SliceND(shape, start, stop, step)); 756 } 757 758 @Override 759 public Dataset getSliceView(Slice... slice) { 760 if (slice == null || slice.length == 0) { 761 return getView(true); 762 } 763 764 return getSliceView(new SliceND(shape, slice)); 765 } 766 767 /** 768 * Get a slice of the dataset. The returned dataset is a view on a selection of items 769 * @param slice 770 * @return slice view 771 */ 772 @Override 773 public Dataset getSliceView(SliceND slice) { 774 if (slice.isAll()) { 775 return getView(true); 776 } 777 778 final int rank = shape.length; 779 int[] sStride = new int[rank]; 780 int[] sOffset = new int[1]; 781 782 int[] sShape = createStrides(slice, this, sStride, sOffset); 783 784 AbstractDataset s = getView(false); 785 s.shape = sShape; 786 s.size = ShapeUtils.calcSize(sShape); 787 s.stride = sStride; 788 s.offset = sOffset[0]; 789 s.base = this; 790 791 s.metadata = copyMetadata(); 792 s.sliceMetadata(true, slice); 793 794 s.setDirty(); 795 s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE); 796 797 return s; 798 } 799 800 /** 801 * Get flattened view index of given position 802 * @param pos 803 * the integer array specifying the n-D position 804 * @return the index on the flattened dataset 805 */ 806 private int getFlat1DIndex(final int[] pos) { 807 final int imax = pos.length; 808 if (imax == 0) { 809 return 0; 810 } 811 812 return get1DIndexFromShape(pos); 813 } 814 815 /** 816 * @since 2.0 817 */ 818 protected int getFirst1DIndex() { 819 if (shape == null) { 820 throw new IllegalArgumentException("Cannot find an index from a null shape"); 821 } 822 return stride == null ? 0 : offset; 823 } 824 825 @Override 826 public int get1DIndex(final int... n) { 827 if (n.length == 0 && shape.length == 0) 828 return offset; 829 830 return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n); 831 } 832 833 private static void throwAIOOBException(int i, int s, int d) { 834 throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s 835 + "] in dimension " + d); 836 } 837 838 /** 839 * @param i 840 * @return the index on the data array corresponding to that location 841 */ 842 protected int get1DIndex(int i) { 843 if (shape == null) { 844 throw new IllegalArgumentException("Cannot find an index from a null shape"); 845 } 846 if (shape.length > 1) { 847 logger.error("This dataset is not 1D but was addressed as such"); 848 throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such"); 849 } 850 if (i < 0) { 851 i += shape[0]; 852 } 853 if (i < 0 || i >= shape[0]) { 854 throwAIOOBException(i, shape[0], 0); 855 } 856 return stride == null ? i : i*stride[0] + offset; 857 } 858 859 /** 860 * @param i 861 * @param j 862 * @return the index on the data array corresponding to that location 863 */ 864 protected int get1DIndex(int i, int j) { 865 if (shape == null) { 866 throw new IllegalArgumentException("Cannot find an index from a null shape"); 867 } 868 if (shape.length != 2) { 869 logger.error("This dataset is not 2D but was addressed as such"); 870 throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such"); 871 } 872 if (i < 0) { 873 i += shape[0]; 874 } 875 if (i < 0 || i >= shape[0]) { 876 throwAIOOBException(i, shape[0], 0); 877 } 878 if (j < 0) { 879 j += shape[1]; 880 } 881 if (j < 0 || j >= shape[1]) { 882 throwAIOOBException(i, shape[1], 1); 883 } 884 return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset; 885 } 886 887 protected int get1DIndexFromShape(final int[] n) { 888 return get1DIndexFromShape(shape, n); 889 } 890 891 protected static int get1DIndexFromShape(final int[] shape, final int[] n) { 892 if (shape == null) { 893 throw new IllegalArgumentException("Cannot find an index from a null shape"); 894 } 895 final int rank = shape.length; 896 if (rank != n.length) { 897 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 898 logger.error(errMsg); 899 throw new IllegalArgumentException(errMsg); 900 } 901 int index = 0; 902 for (int i = 0; i < rank; i++) { 903 final int si = shape[i]; 904 int ni = n[i]; 905 if (ni < 0) { 906 ni += si; 907 } 908 if (ni < 0 || ni >= si) { 909 throwAIOOBException(ni, si, i); 910 } 911 index = index * si + ni; 912 } 913 914 return index; 915 } 916 917 private int get1DIndexFromStrides(final int[] n) { 918 return get1DIndexFromStrides(shape, stride, offset, n); 919 } 920 921 private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) { 922 if (shape == null) { 923 throw new IllegalArgumentException("Cannot find an index from a null shape"); 924 } 925 final int rank = shape.length; 926 if (rank != n.length) { 927 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 928 logger.error(errMsg); 929 throw new IllegalArgumentException(errMsg); 930 } 931 int index = offset; 932 for (int i = 0; i < rank; i++) { 933 final int st = stride[i]; 934 if (st != 0) { // not broadcasted 935 final int si = shape[i]; 936 int ni = n[i]; 937 if (ni < 0) { 938 ni += si; 939 } 940 if (ni < 0 || ni >= si) { 941 throwAIOOBException(ni, si, i); 942 } 943 index += st * ni; 944 } 945 } 946 return index; 947 } 948 949 @Override 950 public int[] getNDPosition(final int n) { 951 if (isIndexInRange(n)) { 952 throw new IllegalArgumentException("Index provided " + n 953 + "is larger then the size of the containing array"); 954 } 955 956 return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n); 957 } 958 959 private boolean isIndexInRange(final int n) { 960 if (stride == null) { 961 return n >= size; 962 } 963 return n >= getBufferLength(); 964 } 965 966 /** 967 * @return entire buffer length 968 */ 969 abstract protected int getBufferLength(); 970 971 private int[] getNDPositionFromStrides(int n) { 972 n -= offset; 973 int rank = shape.length; 974 if (rank == 1) { 975 return new int[] { n / stride[0] }; 976 } 977 978 int[] output = new int[rank]; 979 int i = 0; 980 while (i != n) { // TODO find more efficient way than this exhaustive search 981 int j = rank - 1; 982 for (; j >= 0; j--) { 983 output[j]++; 984 i += stride[j]; 985 if (output[j] >= shape[j]) { 986 output[j] = 0; 987 i -= shape[j] * stride[j]; 988 } else { 989 break; 990 } 991 } 992 if (j == -1) { 993 logger.error("Index was not found in this strided dataset"); 994 throw new IllegalArgumentException("Index was not found in this strided dataset"); 995 } 996 } 997 998 return output; 999 } 1000 1001 @Override 1002 public int checkAxis(int axis) { 1003 return ShapeUtils.checkAxis(shape.length, axis); 1004 } 1005 1006 @Deprecated 1007 protected static int checkAxis(int rank, int axis) { 1008 return ShapeUtils.checkAxis(rank, axis); 1009 } 1010 1011 protected static final char BLOCK_OPEN = '['; 1012 protected static final char BLOCK_CLOSE = ']'; 1013 1014 @Override 1015 public String toString() { 1016 final int rank = shape == null ? 0 : shape.length; 1017 final StringBuilder out = new StringBuilder(); 1018 1019 if (DTypeUtils.isDTypeElemental(getDType())) { 1020 out.append("Dataset "); 1021 } else { 1022 out.append("Compound dataset ("); 1023 out.append(getElementsPerItem()); 1024 out.append(") "); 1025 } 1026 1027 if (name != null && name.length() > 0) { 1028 out.append("'"); 1029 out.append(name); 1030 out.append("' has shape "); 1031 } else { 1032 out.append("shape is "); 1033 } 1034 1035 out.append(BLOCK_OPEN); 1036 if (rank > 0) { 1037 out.append(shape[0]); 1038 } 1039 for (int i = 1; i < rank; i++) { 1040 out.append(", " + shape[i]); 1041 } 1042 out.append(BLOCK_CLOSE); 1043 return out.toString(); 1044 } 1045 1046 @Override 1047 public String toString(boolean showData) { 1048 if (!showData) { 1049 return toString(); 1050 } 1051 1052 if (size == 0) { 1053 return "[]"; 1054 } 1055 1056 final int rank = shape == null ? 0 : shape.length; 1057 final StringBuilder out = new StringBuilder(); 1058 1059 if (rank > 0) { 1060 int[] pos = new int[rank]; 1061 final StringBuilder lead = new StringBuilder(); 1062 printBlocks(out, lead, 0, pos); 1063 } else { 1064 out.append(getString()); 1065 } 1066 return out.toString(); 1067 } 1068 1069 /** 1070 * Limit to strings output via the toString() method 1071 */ 1072 private static int maxStringLength = 120; 1073 1074 /** 1075 * Set maximum line length for toString() method 1076 * @param maxLineLength 1077 */ 1078 public static void setMaxLineLength(int maxLineLength) { 1079 maxStringLength = maxLineLength; 1080 } 1081 1082 /** 1083 * @return maximum line length for toString() method 1084 */ 1085 public static int getMaxLineLength() { 1086 return maxStringLength; 1087 } 1088 1089 /** 1090 * Limit to number of sub-blocks output via the toString() method 1091 */ 1092 private static final int MAX_SUBBLOCKS = 6; 1093 1094 private final static String SEPARATOR = ","; 1095 private final static String SPACE = " "; 1096 private final static String ELLIPSIS = "..."; 1097 private final static String NEWLINE = "\n"; 1098 1099 /** 1100 * Make a line of output for last dimension of dataset 1101 * 1102 * @param start 1103 * @return line 1104 */ 1105 private StringBuilder makeLine(final int end, final int[] start) { 1106 StringBuilder line = new StringBuilder(); 1107 final int[] pos; 1108 if (end >= start.length) { 1109 pos = Arrays.copyOf(start, end + 1); 1110 } else { 1111 pos = start; 1112 } 1113 pos[end] = 0; 1114 line.append(BLOCK_OPEN); 1115 line.append(getString(pos)); 1116 1117 final int length = shape[end]; 1118 1119 // trim elements printed if length exceed estimate of maximum elements 1120 int excess = length - maxStringLength / 3; // space + number + separator 1121 int midIndex = -1; 1122 if (excess > 0) { 1123 int index = (length - excess) / 2; 1124 for (int y = 1; y < index; y++) { 1125 line.append(SEPARATOR + SPACE); 1126 pos[end] = y; 1127 line.append(getString(pos)); 1128 } 1129 midIndex = line.length() + 2; 1130 index = (length + excess) / 2; 1131 for (int y = index; y < length; y++) { 1132 line.append(SEPARATOR + SPACE); 1133 pos[end] = y; 1134 line.append(getString(pos)); 1135 } 1136 } else { 1137 for (int y = 1; y < length; y++) { 1138 line.append(SEPARATOR + SPACE); 1139 pos[end] = y; 1140 line.append(getString(pos)); 1141 } 1142 } 1143 line.append(BLOCK_CLOSE); 1144 1145 // trim string down to limit 1146 int lineLength = line.length(); 1147 excess = lineLength - maxStringLength - ELLIPSIS.length() - 1; 1148 if (excess > 0) { 1149 int index = (lineLength - excess) / 2; 1150 if (midIndex > 0 && index > midIndex) { 1151 index = midIndex; 1152 } else { 1153 index = line.lastIndexOf(SEPARATOR, index) + 2; 1154 } 1155 StringBuilder out = new StringBuilder(line.subSequence(0, index)); 1156 out.append(ELLIPSIS + SEPARATOR); 1157 index = (lineLength + excess) / 2; 1158 if (midIndex > 0 && index <= midIndex) { 1159 index = midIndex - 1; 1160 } else { 1161 index = line.indexOf(SEPARATOR, index) + 1; 1162 } 1163 out.append(line.subSequence(index, lineLength)); 1164 return out; 1165 } else if (midIndex > 0) { // add ellipsis 1166 StringBuilder out = new StringBuilder(line.subSequence(0, midIndex)); 1167 out.append(ELLIPSIS + SEPARATOR + SPACE); 1168 out.append(line.subSequence(midIndex, lineLength)); 1169 return out; 1170 } 1171 1172 return line; 1173 } 1174 1175 /** 1176 * recursive method to print blocks 1177 */ 1178 private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) { 1179 if (out.length() > 0) { 1180 char last = out.charAt(out.length() - 1); 1181 if (last != BLOCK_OPEN) { 1182 out.append(lead); 1183 } 1184 } 1185 final int end = getRank() - 1; 1186 if (level != end) { 1187 out.append(BLOCK_OPEN); 1188 int length = shape[level]; 1189 1190 // first sub-block 1191 pos[level] = 0; 1192 StringBuilder newlead = new StringBuilder(lead); 1193 newlead.append(SPACE); 1194 printBlocks(out, newlead, level + 1, pos); 1195 if (length < 2) { // escape 1196 out.append(BLOCK_CLOSE); 1197 return; 1198 } 1199 1200 out.append(SEPARATOR + NEWLINE); 1201 for (int i = level + 1; i < end; i++) { 1202 out.append(NEWLINE); 1203 } 1204 1205 // middle sub-blocks 1206 if (length < MAX_SUBBLOCKS) { 1207 for (int x = 1; x < length - 1; x++) { 1208 pos[level] = x; 1209 printBlocks(out, newlead, level + 1, pos); 1210 if (end <= level + 1) { 1211 out.append(SEPARATOR + NEWLINE); 1212 } else { 1213 out.append(SEPARATOR + NEWLINE + NEWLINE); 1214 } 1215 } 1216 } else { 1217 final int excess = length - MAX_SUBBLOCKS; 1218 int xmax = (length - excess) / 2; 1219 for (int x = 1; x < xmax; x++) { 1220 pos[level] = x; 1221 printBlocks(out, newlead, level + 1, pos); 1222 if (end <= level + 1) { 1223 out.append(SEPARATOR + NEWLINE); 1224 } else { 1225 out.append(SEPARATOR + NEWLINE + NEWLINE); 1226 } 1227 } 1228 out.append(newlead); 1229 out.append(ELLIPSIS + SEPARATOR + NEWLINE); 1230 xmax = (length + excess) / 2; 1231 for (int x = xmax; x < length - 1; x++) { 1232 pos[level] = x; 1233 printBlocks(out, newlead, level + 1, pos); 1234 if (end <= level + 1) { 1235 out.append(SEPARATOR + NEWLINE); 1236 } else { 1237 out.append(SEPARATOR + NEWLINE + NEWLINE); 1238 } 1239 } 1240 } 1241 1242 // last sub-block 1243 pos[level] = length - 1; 1244 printBlocks(out, newlead, level + 1, pos); 1245 out.append(BLOCK_CLOSE); 1246 } else { 1247 out.append(makeLine(end, pos)); 1248 } 1249 } 1250 1251 @Override 1252 public Dataset squeezeEnds() { 1253 return squeeze(true); 1254 } 1255 1256 @Override 1257 public Dataset squeeze() { 1258 return squeeze(false); 1259 } 1260 1261 @Override 1262 public Dataset squeeze(boolean onlyFromEnds) { 1263 final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds); 1264 final int[] oshape = shape; 1265 if (stride == null) { 1266 shape = tshape; 1267 } else { 1268 int rank = shape.length; 1269 int trank = tshape.length; 1270 if (trank < rank) { 1271 int[] tstride = new int[tshape.length]; 1272 if (onlyFromEnds) { 1273 for (int i = 0; i < rank; i++) { 1274 if (shape[i] != 1) { 1275 for (int k = 0; k < trank; k++) { 1276 tstride[k] = stride[i++]; 1277 } 1278 break; 1279 } 1280 } 1281 } else { 1282 int t = 0; 1283 for (int i = 0; i < rank; i++) { 1284 if (shape[i] != 1) { 1285 tstride[t++] = stride[i]; 1286 } 1287 } 1288 } 1289 shape = tshape; 1290 stride = tstride; 1291 } 1292 } 1293 1294 setDirty(); 1295 reshapeMetadata(oshape, shape); 1296 return this; 1297 } 1298 1299 @Override 1300 public boolean isCompatibleWith(final ILazyDataset g) { 1301 return ShapeUtils.areShapesCompatible(shape, g.getShape()); 1302 } 1303 1304 @Override 1305 public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException { 1306 ShapeUtils.checkCompatibility(this, g); 1307 } 1308 1309 @Override 1310 public Dataset reshape(final int... shape) { 1311 Dataset a; 1312 try { 1313 a = getView(true); 1314 a.setShape(shape); 1315 } catch (IllegalArgumentException e) { 1316 a = clone(); 1317 a.setShape(shape); 1318 } 1319 return a; 1320 } 1321 1322 /** 1323 * @param start 1324 * @param stop 1325 * @param step 1326 * @return number of steps to take 1327 */ 1328 protected static int calcSteps(final double start, final double stop, final double step) { 1329 return Math.max(0, (int) Math.ceil((stop - start) / step)); 1330 } 1331 1332 @Override 1333 public boolean isComplex() { 1334 int type = getDType(); 1335 return type == COMPLEX64 || type == COMPLEX128; 1336 } 1337 1338 @Override 1339 public Dataset getRealPart() { 1340 return this; 1341 } 1342 1343 @Override 1344 public Dataset getRealView() { 1345 return getView(true); 1346 } 1347 1348 @Override 1349 public Dataset getSlice(final int[] start, final int[] stop, final int[] step) { 1350 return getSlice(new SliceND(shape, start, stop, step)); 1351 } 1352 1353 @Override 1354 public Dataset getSlice(Slice... slice) { 1355 return getSlice(new SliceND(shape, slice)); 1356 } 1357 1358 @Override 1359 public Dataset getSlice(IMonitor monitor, Slice... slice) { 1360 return getSlice(slice); 1361 } 1362 1363 @Override 1364 public Dataset getSlice(IMonitor monitor, SliceND slice) { 1365 return getSlice(slice); 1366 } 1367 1368 @Override 1369 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) { 1370 return getSlice(start, stop, step); 1371 } 1372 1373 /** 1374 * Get a slice of the dataset. The returned dataset is a copied selection of items 1375 * @param slice 1376 * @return The dataset of the sliced data 1377 */ 1378 @Override 1379 public Dataset getSlice(final SliceND slice) { 1380 SliceIterator it = (SliceIterator) getSliceIterator(slice); 1381 AbstractDataset s = getSlice(it); 1382 s.metadata = copyMetadata(); 1383 s.setDirty(); 1384 s.sliceMetadata(true, slice); 1385 return s; 1386 } 1387 1388 /** 1389 * Get a slice of the dataset. The returned dataset is a copied selection of items 1390 * 1391 * @param iterator Slice iterator 1392 * @return The dataset of the sliced data 1393 */ 1394 abstract public AbstractDataset getSlice(final SliceIterator iterator); 1395 1396 @SuppressWarnings("deprecation") 1397 @Override 1398 public Dataset setSlice(final Object obj, final SliceND slice) { 1399 Dataset ds; 1400 if (obj instanceof Dataset) { 1401 ds = (Dataset) obj; 1402 } else if (obj instanceof IDataset) { 1403 ds = DatasetUtils.convertToDataset((IDataset) obj); 1404 } else { 1405 int dtype = getDType(); 1406 if (dtype != Dataset.BOOL) { 1407 dtype = DTypeUtils.getLargestDType(dtype); 1408 } 1409 ds = DatasetFactory.createFromObject(getElementsPerItem(), dtype, obj); 1410 } 1411 1412 return setSlicedView(getSliceView(slice), ds); 1413 } 1414 1415 @Override 1416 public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) { 1417 return setSlice(obj, new SliceND(shape, start, stop, step)); 1418 } 1419 1420 /** 1421 * Set a view of current dataset to given dataset with broadcasting 1422 * @param view 1423 * @param d 1424 * @return this dataset 1425 */ 1426 abstract Dataset setSlicedView(Dataset view, Dataset d); 1427 1428 @Override 1429 public Dataset setSlice(Object obj, Slice... slice) { 1430 if (slice == null || slice.length == 0) { 1431 return setSlice(obj, new SliceND(shape)); 1432 } 1433 return setSlice(obj, new SliceND(shape, slice)); 1434 } 1435 1436 @Override 1437 public boolean all() { 1438 return Comparisons.allTrue(this); 1439 } 1440 1441 @Override 1442 public BooleanDataset all(final int axis) { 1443 return Comparisons.allTrue(this, axis); 1444 } 1445 1446 @Override 1447 public boolean any() { 1448 return Comparisons.anyTrue(this); 1449 } 1450 1451 @Override 1452 public BooleanDataset any(final int axis) { 1453 return Comparisons.anyTrue(this, axis); 1454 } 1455 1456 @Override 1457 public Dataset ifloorDivide(final Object o) { 1458 return idivide(o).ifloor(); 1459 } 1460 1461 @Override 1462 public double residual(final Object o) { 1463 return residual(o, null, false); 1464 } 1465 1466 @Override 1467 public double residual(final Object o, boolean ignoreNaNs) { 1468 return residual(o, null, ignoreNaNs); 1469 } 1470 1471 /** 1472 * @since 2.0 1473 */ 1474 @SuppressWarnings("unchecked") 1475 protected StatisticsMetadata<Number> getStats() { 1476 StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class); 1477 if (md == null || md.isDirty(this)) { 1478 md = new StatisticsMetadataImpl<Number>(); 1479 md.initialize(this); 1480 setMetadata(md); 1481 } 1482 return md; 1483 } 1484 1485 /** 1486 * @since 2.0 1487 */ 1488 @SuppressWarnings("unchecked") 1489 protected StatisticsMetadata<String> getStringStats() { 1490 StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class); 1491 if (md == null || md.isDirty(this)) { 1492 md = new StatisticsMetadataImpl<String>(); 1493 md.initialize(this); 1494 setMetadata(md); 1495 } 1496 return md; 1497 } 1498 1499 @Override 1500 public Number max(boolean... ignoreInvalids) { 1501 return getStats().getMaximum(ignoreInvalids); 1502 } 1503 1504 @Override 1505 public Dataset max(int axis, boolean... ignoreInvalids) { 1506 return getStats().getMaximum(axis, ignoreInvalids); 1507 } 1508 1509 @Override 1510 public Dataset max(int[] axes, boolean... ignoreInvalids) { 1511 return getStats().getMaximum(axes, ignoreInvalids); 1512 } 1513 1514 @Override 1515 public Number min(boolean... ignoreInvalids) { 1516 return getStats().getMinimum(ignoreInvalids); 1517 } 1518 1519 @Override 1520 public Dataset min(int axis, boolean... ignoreInvalids) { 1521 return getStats().getMinimum(axis, ignoreInvalids); 1522 } 1523 1524 @Override 1525 public Dataset min(int[] axes, boolean... ignoreInvalids) { 1526 return getStats().getMinimum(axes, ignoreInvalids); 1527 } 1528 1529 @Override 1530 public int argMax(boolean... ignoreInvalids) { 1531 return getFlat1DIndex(maxPos(ignoreInvalids)); 1532 } 1533 1534 /** 1535 * @since 2.0 1536 */ 1537 @Override 1538 public IntegerDataset argMax(int axis, boolean... ignoreInvalids) { 1539 return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids); 1540 } 1541 1542 @Override 1543 public int argMin(boolean... ignoreInvalids) { 1544 return getFlat1DIndex(minPos(ignoreInvalids)); 1545 } 1546 1547 /** 1548 * @since 2.0 1549 */ 1550 @Override 1551 public IntegerDataset argMin(int axis, boolean... ignoreInvalids) { 1552 return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids); 1553 } 1554 1555 @Override 1556 public Number peakToPeak(boolean... ignoreInvalids) { 1557 return DTypeUtils.fromDoubleToBiggestNumber(max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue(), getDType()); 1558 } 1559 1560 @Override 1561 public Dataset peakToPeak(int axis, boolean... ignoreInvalids) { 1562 return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids)); 1563 } 1564 1565 @Override 1566 public Dataset peakToPeak(int[] axes, boolean... ignoreInvalids) { 1567 return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids)); 1568 } 1569 1570 @Override 1571 public long count(boolean... ignoreInvalids) { 1572 return getStats().getCount(ignoreInvalids); 1573 } 1574 1575 @Override 1576 public Dataset count(int axis, boolean... ignoreInvalids) { 1577 return getStats().getCount(axis, ignoreInvalids); 1578 } 1579 1580 @Override 1581 public Dataset count(int[] axes, boolean... ignoreInvalids) { 1582 return getStats().getCount(axes, ignoreInvalids); 1583 } 1584 1585 @Override 1586 public Object sum(boolean... ignoreInvalids) { 1587 return getStats().getSum(ignoreInvalids); 1588 } 1589 1590 @Override 1591 public Dataset sum(int axis, boolean... ignoreInvalids) { 1592 return getStats().getSum(axis, ignoreInvalids); 1593 } 1594 1595 @Override 1596 public Dataset sum(int[] axes, boolean... ignoreInvalids) { 1597 return getStats().getSum(axes, ignoreInvalids); 1598 } 1599 1600 @Override 1601 public Object product(boolean... ignoreInvalids) { 1602 return Stats.product(this, ignoreInvalids); 1603 } 1604 1605 @Override 1606 public Dataset product(int axis, boolean... ignoreInvalids) { 1607 return Stats.product(this, axis, ignoreInvalids); 1608 } 1609 1610 @Override 1611 public Dataset product(int[] axes, boolean... ignoreInvalids) { 1612 return Stats.product(this, axes, ignoreInvalids); 1613 } 1614 1615 @Override 1616 public Object mean(boolean... ignoreInvalids) { 1617 return getStats().getMean(ignoreInvalids); 1618 } 1619 1620 @Override 1621 public Dataset mean(int axis, boolean... ignoreInvalids) { 1622 return getStats().getMean(axis, ignoreInvalids); 1623 } 1624 1625 @Override 1626 public Dataset mean(int[] axes, boolean... ignoreInvalids) { 1627 return getStats().getMean(axes, ignoreInvalids); 1628 } 1629 1630 @Override 1631 public double variance() { 1632 return variance(false); 1633 } 1634 1635 @Override 1636 public double variance(boolean isWholePopulation, boolean... ignoreInvalids) { 1637 return getStats().getVariance(isWholePopulation, ignoreInvalids); 1638 } 1639 1640 @Override 1641 public Dataset variance(int axis) { 1642 return getStats().getVariance(axis, false); 1643 } 1644 1645 @Override 1646 public Dataset variance(int[] axes) { 1647 return getStats().getVariance(axes, false); 1648 } 1649 1650 @Override 1651 public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1652 return getStats().getVariance(axis, isWholePopulation, ignoreInvalids); 1653 } 1654 1655 @Override 1656 public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) { 1657 return getStats().getVariance(axes, isWholePopulation, ignoreInvalids); 1658 } 1659 1660 @Override 1661 public double stdDeviation() { 1662 return Math.sqrt(variance()); 1663 } 1664 1665 @Override 1666 public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) { 1667 return Math.sqrt(variance(isWholePopulation, ignoreInvalids)); 1668 } 1669 1670 @Override 1671 public Dataset stdDeviation(int axis) { 1672 return Maths.sqrt(variance(axis, false)); 1673 } 1674 1675 @Override 1676 public Dataset stdDeviation(int[] axes) { 1677 return Maths.sqrt(variance(axes, false)); 1678 } 1679 1680 @Override 1681 public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1682 return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids)); 1683 } 1684 1685 @Override 1686 public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) { 1687 return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids)); 1688 } 1689 1690 @Override 1691 public double rootMeanSquare(boolean... ignoreInvalids) { 1692 StatisticsMetadata<Number> stats = getStats(); 1693 final double mean = stats.getMean(ignoreInvalids).doubleValue(); 1694 final double var = stats.getVariance(true, ignoreInvalids); 1695 return Math.sqrt(var + mean * mean); 1696 } 1697 1698 @Override 1699 public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) { 1700 StatisticsMetadata<Number> stats = getStats(); 1701 Dataset v = stats.getVariance(axis, true, ignoreInvalids); 1702 Dataset m = stats.getMean(axis, ignoreInvalids); 1703 Dataset result = Maths.multiply(m, m); 1704 return Maths.sqrt(result.iadd(v)); 1705 } 1706 1707 @Override 1708 public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) { 1709 StatisticsMetadata<Number> stats = getStats(); 1710 Dataset v = stats.getVariance(axes, true, ignoreInvalids); 1711 Dataset m = stats.getMean(axes, ignoreInvalids); 1712 Dataset result = Maths.multiply(m, m); 1713 return Maths.sqrt(result.iadd(v)); 1714 } 1715 1716 /** 1717 * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards. 1718 * 1719 * @param dindex 1720 * @param sindex 1721 * @param src 1722 * is the source data buffer 1723 */ 1724 protected abstract void setItemDirect(final int dindex, final int sindex, final Object src); 1725 1726 /** 1727 * @return error broadcasted to current shape 1728 */ 1729 private Dataset getBroadcastedInternalError() { 1730 if (shape == null) { 1731 throw new IllegalArgumentException("Cannot get error for null dataset"); 1732 } 1733 ILazyDataset led = super.getErrors(); 1734 if (led == null) 1735 return null; 1736 1737 Dataset ed = null; 1738 try { 1739 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1740 } catch (DatasetException e) { 1741 logger.error("Could not get data from lazy dataset", e); 1742 } 1743 if (led != ed) { 1744 setErrors(ed); // set back 1745 } 1746 1747 return ed.getBroadcastView(shape); 1748 } 1749 1750 @Override 1751 public Dataset getErrors() { 1752 Dataset ed = getBroadcastedInternalError(); 1753 if (ed == null) 1754 return null; 1755 1756 return ed; 1757 } 1758 1759 @Override 1760 public double getError() { 1761 Dataset ed = getBroadcastedInternalError(); 1762 if (ed == null) 1763 return 0; 1764 1765 return ed.getDouble(); 1766 } 1767 1768 @Override 1769 public double getError(final int i) { 1770 Dataset ed = getBroadcastedInternalError(); 1771 if (ed == null) 1772 return 0; 1773 1774 return ed.getDouble(i); 1775 } 1776 1777 @Override 1778 public double getError(final int i, final int j) { 1779 Dataset ed = getBroadcastedInternalError(); 1780 if (ed == null) 1781 return 0; 1782 1783 return ed.getDouble(i, j); 1784 } 1785 1786 @Override 1787 public double getError(int... pos) { 1788 Dataset ed = getBroadcastedInternalError(); 1789 if (ed == null) 1790 return 0; 1791 1792 return ed.getDouble(pos); 1793 } 1794 1795 @Override 1796 public double[] getErrorArray(final int i) { 1797 Dataset ed = getBroadcastedInternalError(); 1798 if (ed == null) 1799 return null; 1800 1801 return new double[] {getError(i)}; 1802 } 1803 1804 @Override 1805 public double[] getErrorArray(final int i, final int j) { 1806 Dataset ed = getBroadcastedInternalError(); 1807 if (ed == null) 1808 return null; 1809 1810 return new double[] {getError(i, j)}; 1811 } 1812 1813 @Override 1814 public double[] getErrorArray(int... pos) { 1815 Dataset ed = getBroadcastedInternalError(); 1816 if (ed == null) 1817 return null; 1818 1819 return new double[] {getError(pos)}; 1820 } 1821 1822 protected Dataset getInternalSquaredError() { 1823 Dataset sed = getErrorBuffer().getBroadcastView(shape); 1824 return sed; 1825 } 1826 1827 @Override 1828 public Dataset getErrorBuffer() { 1829 ErrorMetadata emd = getErrorMetadata(); 1830 if (emd == null) 1831 return null; 1832 1833 if (!(emd instanceof ErrorMetadataImpl)) { 1834 ILazyDataset led = emd.getError(); 1835 Dataset ed; 1836 try { 1837 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1838 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1839 setMetadata(emd); 1840 emd.setError(ed); 1841 } catch (MetadataException me) { 1842 logger.error("Could not create metadata", me); 1843 } catch (DatasetException e) { 1844 logger.error("Could not get data from lazy dataset", e); 1845 } 1846 } 1847 1848 return ((ErrorMetadataImpl) emd).getSquaredError(); 1849 } 1850 1851 /** 1852 * Set a copy of the buffer that backs the (squared) error data 1853 * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset 1854 */ 1855 @Override 1856 public void setErrorBuffer(Serializable buffer) { 1857 if (shape == null) { 1858 throw new IllegalArgumentException("Cannot set error buffer for null dataset"); 1859 } 1860 if (buffer == null) { 1861 clearMetadata(ErrorMetadata.class); 1862 return; 1863 } 1864 1865 IDataset d = (IDataset) createFromSerializable(buffer, false); 1866 ErrorMetadata emd = getErrorMetadata(); 1867 if (!(emd instanceof ErrorMetadataImpl)) { 1868 try { 1869 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1870 setMetadata(emd); 1871 } catch (MetadataException me) { 1872 logger.error("Could not create metadata", me); 1873 } 1874 } 1875 ((ErrorMetadataImpl) emd).setSquaredError(d); 1876 } 1877}