001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.util.Arrays; 016 017import org.eclipse.january.DatasetException; 018import org.eclipse.january.IMonitor; 019import org.eclipse.january.metadata.StatisticsMetadata; 020import org.eclipse.january.metadata.internal.StatisticsMetadataImpl; 021import org.slf4j.Logger; 022import org.slf4j.LoggerFactory; 023 024/** 025 * Generic container class for data that is compound in nature 026 * 027 * Each subclass has an array of compound types, items of this array are composed of primitive types 028 * 029 * Data items can be Complex, Vector, etc 030 * 031 */ 032public abstract class AbstractCompoundDataset extends AbstractDataset implements CompoundDataset { 033 // pin UID to base class 034 private static final long serialVersionUID = Dataset.serialVersionUID; 035 036 private static final Logger logger = LoggerFactory.getLogger(AbstractCompoundDataset.class); 037 038 protected int isize; // number of elements per item 039 040 @Override 041 public int getElementsPerItem() { 042 return isize; 043 } 044 045 @Override 046 protected int get1DIndex(final int i) { 047 int n = super.get1DIndex(i); 048 return stride == null ? isize * n : n; 049 } 050 051 @Override 052 protected int get1DIndex(final int i, final int j) { 053 int n = super.get1DIndex(i, j); 054 return stride == null ? isize * n : n; 055 } 056 057 @Override 058 protected int get1DIndexFromShape(final int[] n) { 059 return isize * super.get1DIndexFromShape(n); 060 } 061 062 @Override 063 public Dataset getUniqueItems() { 064 throw new UnsupportedOperationException("Cannot sort compound datasets"); 065 } 066 067 @Override 068 public IndexIterator getIterator(final boolean withPosition) { 069 if (stride != null) { 070 return base.getSize() == 1 ? 071 (withPosition ? new PositionIterator(offset, shape) : new SingleItemIterator(offset, size)) : new StrideIterator(isize, shape, stride, offset); 072 } 073 return withPosition ? getSliceIterator(null, null, null) : 074 new ContiguousIterator(size, isize); 075 } 076 077 /** 078 * Get an iterator that picks out the chosen element from all items 079 * @param element to choose 080 * @return an iterator 081 */ 082 public IndexIterator getIterator(int element) { 083 if (element < 0) 084 element += isize; 085 if (element < 0 || element > isize) { 086 logger.error("Invalid choice of element: {}/{}", element, isize); 087 throw new IllegalArgumentException("Invalid choice of element: " + element + "/" + isize); 088 } 089 090 final IndexIterator it; 091 if (stride != null) { 092 it = base.getSize() == 1 ? new SingleItemIterator(offset + element, size) : new StrideIterator(isize, shape, stride, offset, element); 093 } else { 094 it = new ContiguousIterator(size, isize, element); 095 } 096 097 return it; 098 } 099 100 /** 101 * @param slice to define iterator 102 * @return an slice iterator that operates like an IndexIterator 103 */ 104 @Override 105 protected IndexIterator internalGetSliceIterator(SliceND slice) { 106 if (ShapeUtils.calcLongSize(slice.getShape()) == 0) { 107 return new NullIterator(shape, slice.getShape()); 108 } 109 if (stride != null) { 110 return new StrideIterator(isize, shape, stride, offset, slice); 111 } 112 113 return new SliceIterator(shape, size, isize, slice); 114 } 115 116 /** 117 * Constructor required for serialisation. 118 */ 119 public AbstractCompoundDataset() { 120 } 121 122 @Override 123 public boolean equals(Object obj) { 124 if (!super.equals(obj)) { 125 return false; 126 } 127 128 CompoundDataset other = (CompoundDataset) obj; 129 return isize == other.getElementsPerItem(); 130 } 131 132 @Override 133 public int hashCode() { 134 return getCompoundStats().getHash(shape); 135 } 136 137 @Override 138 public CompoundDataset cast(boolean repeat, int dtype, int isize) { 139 return (CompoundDataset) super.cast(repeat, dtype, isize); 140 } 141 142 @Override 143 public CompoundDataset cast(int dtype) { 144 return (CompoundDataset) super.cast(dtype); 145 } 146 147 @Override 148 abstract public AbstractCompoundDataset clone(); 149 150 @Override 151 public CompoundDataset flatten() { 152 return (CompoundDataset) super.flatten(); 153 } 154 155 @Override 156 public CompoundDataset getBy1DIndex(IntegerDataset index) { 157 return (CompoundDataset) super.getBy1DIndex(index); 158 } 159 160 @Override 161 public CompoundDataset getByBoolean(Dataset selection) { 162 return (CompoundDataset) super.getByBoolean(selection); 163 } 164 165 @Override 166 public CompoundDataset getByIndexes(Object... indexes) { 167 return (CompoundDataset) super.getByIndexes(indexes); 168 } 169 170 @Override 171 public CompoundDataset getSlice(IMonitor mon, int[] start, int[] stop, int[] step) { 172 return (CompoundDataset) super.getSlice(mon, start, stop, step); 173 } 174 175 @Override 176 public CompoundDataset getSlice(IMonitor mon, Slice... slice) { 177 return (CompoundDataset) super.getSlice(mon, slice); 178 } 179 180 @Override 181 public CompoundDataset getSlice(IMonitor mon, SliceND slice) { 182 return (CompoundDataset) super.getSlice(mon, slice); 183 } 184 185 @Override 186 public CompoundDataset getSlice(int[] start, int[] stop, int[] step) { 187 return (CompoundDataset) super.getSlice(start, stop, step); 188 } 189 190 @Override 191 public CompoundDataset getSlice(Slice... slice) { 192 return (CompoundDataset) super.getSlice(slice); 193 } 194 195 @Override 196 public CompoundDataset getSlice(SliceND slice) { 197 return (CompoundDataset) super.getSlice(slice); 198 } 199 200 @Override 201 abstract public AbstractCompoundDataset getSlice(SliceIterator iterator); 202 203 @Override 204 public CompoundDataset getSliceView(int[] start, int[] stop, int[] step) { 205 return (CompoundDataset) super.getSliceView(start, stop, step); 206 } 207 208 @Override 209 public CompoundDataset getSliceView(Slice... slice) { 210 return (CompoundDataset) super.getSliceView(slice); 211 } 212 213 @Override 214 public CompoundDataset getSliceView(SliceND slice) { 215 return (CompoundDataset) super.getSliceView(slice); 216 } 217 218 @Override 219 public CompoundDataset getTransposedView(int... axes) { 220 return (CompoundDataset) super.getTransposedView(axes); 221 } 222 223 @Override 224 abstract public AbstractCompoundDataset getView(boolean deepCopyMetadata); 225 226 @Override 227 public CompoundDataset getBroadcastView(int... broadcastShape) { 228 return (CompoundDataset) super.getBroadcastView(broadcastShape); 229 } 230 231 @Override 232 public CompoundDataset ifloorDivide(Object o) { 233 return (CompoundDataset) super.ifloorDivide(o); 234 } 235 236 @Override 237 public CompoundDataset reshape(int... shape) { 238 return (CompoundDataset) super.reshape(shape); 239 } 240 241 @Override 242 public CompoundDataset setSlice(Object obj, int[] start, int[] stop, int[] step) { 243 return (CompoundDataset) super.setSlice(obj, start, stop, step); 244 } 245 246 @Override 247 public CompoundDataset setSlice(Object object, Slice... slice) { 248 return (CompoundDataset) super.setSlice(object, slice); 249 } 250 251 @Override 252 public CompoundDataset sort(Integer axis) { 253 throw new UnsupportedOperationException("Cannot sort dataset"); 254 } 255 256 @Override 257 public CompoundDataset squeezeEnds() { 258 return (CompoundDataset) super.squeezeEnds(); 259 } 260 261 @Override 262 public CompoundDataset squeeze() { 263 return (CompoundDataset) super.squeeze(); 264 } 265 266 @Override 267 public CompoundDataset squeeze(boolean onlyFromEnd) { 268 return (CompoundDataset) super.squeeze(onlyFromEnd); 269 } 270 271 @Override 272 public CompoundDataset swapAxes(int axis1, int axis2) { 273 return (CompoundDataset) super.swapAxes(axis1, axis2); 274 } 275 276 @Override 277 public synchronized CompoundDataset synchronizedCopy() { 278 return clone(); 279 } 280 281 @Override 282 public CompoundDataset transpose(int... axes) { 283 return (CompoundDataset) super.transpose(axes); 284 } 285 286 /** 287 * @since 2.0 288 * @return first value 289 */ 290 abstract protected double getFirstValue(); 291 292 abstract protected double getFirstValue(final int i); 293 294 abstract protected double getFirstValue(final int i, final int j); 295 296 abstract protected double getFirstValue(final int...pos); 297 298 @Override 299 public boolean getBoolean() { 300 return getFirstValue() != 0; 301 } 302 303 @Override 304 public boolean getBoolean(final int i) { 305 return getFirstValue(i) != 0; 306 } 307 308 @Override 309 public boolean getBoolean(final int i, final int j) { 310 return getFirstValue(i, j) != 0; 311 } 312 313 @Override 314 public boolean getBoolean(final int... pos) { 315 return getFirstValue(pos) != 0; 316 } 317 318 @Override 319 public byte getByte() { 320 return (byte) getFirstValue(); 321 } 322 323 @Override 324 public byte getByte(final int i) { 325 return (byte) getFirstValue(i); 326 } 327 328 @Override 329 public byte getByte(final int i, final int j) { 330 return (byte) getFirstValue(i, j); 331 } 332 333 @Override 334 public byte getByte(final int... pos) { 335 return (byte) getFirstValue(pos); 336 } 337 338 @Override 339 public short getShort() { 340 return (short) getFirstValue(); 341 } 342 343 @Override 344 public short getShort(final int i) { 345 return (short) getFirstValue(i); 346 } 347 348 @Override 349 public short getShort(final int i, final int j) { 350 return (short) getFirstValue(i, j); 351 } 352 353 @Override 354 public short getShort(final int... pos) { 355 return (short) getFirstValue(pos); 356 } 357 358 @Override 359 public int getInt() { 360 return (int) getFirstValue(); 361 } 362 363 @Override 364 public int getInt(final int i) { 365 return (int) getFirstValue(i); 366 } 367 368 @Override 369 public int getInt(final int i, final int j) { 370 return (int) getFirstValue(i, j); 371 } 372 373 @Override 374 public int getInt(final int... pos) { 375 return (int) getFirstValue(pos); 376 } 377 378 @Override 379 public long getLong() { 380 return (long) getFirstValue(); 381 } 382 383 @Override 384 public long getLong(final int i) { 385 return (long) getFirstValue(i); 386 } 387 388 @Override 389 public long getLong(final int i, final int j) { 390 return (long) getFirstValue(i, j); 391 } 392 393 @Override 394 public long getLong(final int... pos) { 395 return (long) getFirstValue(pos); 396 } 397 398 @Override 399 public float getFloat() { 400 return (float) getFirstValue(); 401 } 402 403 @Override 404 public float getFloat(final int i) { 405 return (float) getFirstValue(i); 406 } 407 408 @Override 409 public float getFloat(final int i, final int j) { 410 return (float) getFirstValue(i, j); 411 } 412 413 @Override 414 public float getFloat(final int... pos) { 415 return (float) getFirstValue(pos); 416 } 417 418 @Override 419 public double getDouble() { 420 return getFirstValue(); 421 } 422 423 @Override 424 public double getDouble(final int i) { 425 return getFirstValue(i); 426 } 427 428 @Override 429 public double getDouble(final int i, final int j) { 430 return getFirstValue(i, j); 431 } 432 433 @Override 434 public double getDouble(final int... pos) { 435 return getFirstValue(pos); 436 } 437 438 @Override 439 public void getDoubleArray(final double[] darray) { 440 getDoubleArrayAbs(getFirst1DIndex(), darray); 441 } 442 443 @Override 444 public void getDoubleArray(final double[] darray, final int i) { 445 getDoubleArrayAbs(get1DIndex(i), darray); 446 } 447 448 @Override 449 public void getDoubleArray(final double[] darray, final int i, final int j) { 450 getDoubleArrayAbs(get1DIndex(i, j), darray); 451 } 452 453 @Override 454 public void getDoubleArray(final double[] darray, final int... pos) { 455 getDoubleArrayAbs(get1DIndex(pos), darray); 456 } 457 458 /** 459 * @return statistics metadata 460 * @since 2.0 461 */ 462 @SuppressWarnings("unchecked") 463 protected StatisticsMetadata<double[]> getCompoundStats() { 464 StatisticsMetadata<double[]> md = getFirstMetadata(StatisticsMetadata.class); 465 if (md == null || md.isDirty(this)) { 466 md = new StatisticsMetadataImpl<double[]>(); 467 md.initialize(this); 468 setMetadata(md); 469 } 470 return md; 471 } 472 473 @Override 474 public IntegerDataset argMax(int axis, boolean... ignoreInvalids) { 475 logger.error("Cannot compare compound numbers"); 476 throw new UnsupportedOperationException("Cannot compare compound numbers"); 477 } 478 479 @Override 480 public IntegerDataset argMin(int axis, boolean... ignoreInvalids) { 481 logger.error("Cannot compare compound numbers"); 482 throw new UnsupportedOperationException("Cannot compare compound numbers"); 483 } 484 485 @Override 486 public Number max(boolean... ignoreInvalids) { 487 logger.error("Cannot compare compound numbers"); 488 throw new UnsupportedOperationException("Cannot compare compound numbers"); 489 } 490 491 @Override 492 public CompoundDataset max(int axis, boolean... ignoreInvalids) { 493 logger.error("Cannot compare compound numbers"); 494 throw new UnsupportedOperationException("Cannot compare compound numbers"); 495 } 496 497 @Override 498 public Number min(boolean... ignoreInvalids) { 499 logger.error("Cannot compare compound numbers"); 500 throw new UnsupportedOperationException("Cannot compare compound numbers"); 501 } 502 503 @Override 504 public CompoundDataset min(int axis, boolean... ignoreInvalids) { 505 logger.error("Cannot compare compound numbers"); 506 throw new UnsupportedOperationException("Cannot compare compound numbers"); 507 } 508 509 510 @Override 511 public int[] maxPos(boolean... ignoreNaNs) { 512 logger.error("Cannot compare compound numbers"); 513 throw new UnsupportedOperationException("Cannot compare compound numbers"); 514 } 515 516 @Override 517 public int[] minPos(boolean... ignoreNaNs) { 518 logger.error("Cannot compare compound numbers"); 519 throw new UnsupportedOperationException("Cannot compare compound numbers"); 520 } 521 522 @Override 523 public CompoundDataset peakToPeak(int axis, boolean... ignoreInvalids) { 524 logger.error("Cannot compare compound numbers"); 525 throw new UnsupportedOperationException("Cannot compare compound numbers"); 526 } 527 528 @Override 529 public double[] maxItem() { 530 return getCompoundStats().getMaximum(); 531 } 532 533 @Override 534 public double[] minItem() { 535 return getCompoundStats().getMinimum(); 536 } 537 538 @Override 539 public Object mean(boolean... ignoreInvalids) { 540 return getCompoundStats().getMean(); 541 } 542 543 @Override 544 public CompoundDataset mean(int axis, boolean... ignoreInvalids) { 545 return (CompoundDataset) super.mean(axis, ignoreInvalids); 546 } 547 548 @Override 549 public CompoundDataset product(int axis, boolean... ignoreInvalids) { 550 return (CompoundDataset) super.product(axis, ignoreInvalids); 551 } 552 553 @Override 554 public CompoundDataset rootMeanSquare(int axis, boolean... ignoreInvalids) { 555 return (CompoundDataset) super.rootMeanSquare(axis, ignoreInvalids); 556 } 557 558 @Override 559 public CompoundDataset stdDeviation(int axis) { 560 return (CompoundDataset) super.stdDeviation(axis, false); 561 } 562 563 @Override 564 public CompoundDataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 565 return (CompoundDataset) super.stdDeviation(axis, isWholePopulation, ignoreInvalids); 566 } 567 568 @Override 569 public Object sum(boolean... ignoreInvalids) { 570 return getCompoundStats().getSum(); 571 } 572 573 @Override 574 public CompoundDataset sum(int axis, boolean... ignoreInvalids) { 575 return (CompoundDataset) super.sum(axis, ignoreInvalids); 576 } 577 578 @Override 579 public double variance(boolean isWholePopulation, boolean... ignoreInvalids) { 580 return getCompoundStats().getVariance(isWholePopulation, ignoreInvalids); 581 } 582 583 @Override 584 public CompoundDataset variance(int axis) { 585 return (CompoundDataset) super.variance(axis, false); 586 } 587 588 @Override 589 public CompoundDataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 590 return (CompoundDataset) super.variance(axis, isWholePopulation, ignoreInvalids); 591 } 592 593 @Override 594 public double rootMeanSquare(boolean... ignoreInvalids) { 595 StatisticsMetadata<double[]> stats = getCompoundStats(); 596 597 double[] mean = stats.getMean(ignoreInvalids); 598 double result = 0; 599 for (int i = 0; i < isize; i++) { 600 double m = mean[i]; 601 result += m * m; 602 } 603 return Math.sqrt(result + stats.getVariance(true)); 604 } 605 606 /** 607 * @return error 608 */ 609 private CompoundDataset getInternalError() { 610 ILazyDataset led = super.getErrors(); 611 if (led == null) 612 return null; 613 614 Dataset ed = null; 615 try { 616 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 617 } catch (DatasetException e) { 618 logger.error("Could not get data from lazy dataset", e); 619 } 620 621 CompoundDataset ced; // ensure it has the same number of elements 622 if (!(ed instanceof CompoundDataset) || ed.getElementsPerItem() != isize) { 623 ced = new CompoundDoubleDataset(isize, true, ed); 624 } else { 625 ced = (CompoundDataset) ed; 626 } 627 628 if (led != ced) { 629 setErrors(ced); // set back 630 } 631 return ced; 632 } 633 634 @Override 635 public CompoundDataset getErrors() { 636 CompoundDataset ed = getInternalError(); 637 if (ed == null) 638 return null; 639 640 return ed.getBroadcastView(shape); 641 } 642 643 @Override 644 public double getError(final int i) { 645 return calcError(getInternalErrorArray(true, i)); 646 } 647 648 @Override 649 public double getError(final int i, final int j) { 650 return calcError(getInternalErrorArray(true, i, j)); 651 } 652 653 @Override 654 public double getError(final int... pos) { 655 return calcError(getInternalErrorArray(true, pos)); 656 } 657 658 private double calcError(double[] es) { 659 if (es == null) 660 return 0; 661 662 // assume elements are independent 663 double e = 0; 664 for (int k = 0; k < isize; k++) { 665 e += es[k]; 666 } 667 668 return Math.sqrt(e); 669 } 670 671 @Override 672 public double[] getErrorArray(final int i) { 673 return getInternalErrorArray(false, i); 674 } 675 676 @Override 677 public double[] getErrorArray(final int i, final int j) { 678 return getInternalErrorArray(false, i, j); 679 } 680 681 @Override 682 public double[] getErrorArray(final int... pos) { 683 return getInternalErrorArray(false, pos); 684 } 685 686 private Dataset getInternalError(final boolean squared) { 687 Dataset sed = squared ? getInternalSquaredError() : getInternalError(); 688 if (sed == null) 689 return null; 690 691 return sed.getBroadcastView(shape); 692 } 693 694 private double[] getInternalErrorArray(final boolean squared, final int i) { 695 Dataset sed = getInternalError(squared); 696 if (sed == null) 697 return null; 698 699 double[] es; 700 if (sed instanceof CompoundDoubleDataset) { 701 es = ((CompoundDoubleDataset) sed).getDoubleArray(i); 702 if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted 703 Arrays.fill(es, es[0]); 704 } 705 } else { 706 es = new double[isize]; 707 Arrays.fill(es, ((DoubleDataset) sed).getDouble(i)); 708 } 709 return es; 710 } 711 712 private double[] getInternalErrorArray(final boolean squared, final int i, final int j) { 713 Dataset sed = getInternalError(squared); 714 if (sed == null) 715 return null; 716 717 double[] es; 718 if (sed instanceof CompoundDoubleDataset) { 719 es = ((CompoundDoubleDataset) sed).getDoubleArray(i, j); 720 if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted 721 Arrays.fill(es, es[0]); 722 } 723 } else { 724 es = new double[isize]; 725 Arrays.fill(es, ((DoubleDataset) sed).getDouble(i, j)); 726 } 727 return es; 728 } 729 730 private double[] getInternalErrorArray(final boolean squared, final int... pos) { 731 Dataset sed = getInternalError(squared); 732 if (sed == null) 733 return null; 734 735 double[] es = new double[isize]; 736 if (sed instanceof CompoundDoubleDataset) { 737 es = ((CompoundDoubleDataset) sed).getDoubleArray(pos); 738 if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted 739 Arrays.fill(es, es[0]); 740 } 741 } else { 742 es = new double[isize]; 743 Arrays.fill(es, ((DoubleDataset) sed).getDouble(pos)); 744 } 745 return es; 746 } 747} 748