001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.IOException;
016import java.io.Serializable;
017import java.lang.annotation.Annotation;
018import java.lang.reflect.Field;
019import java.util.ArrayList;
020import java.util.Arrays;
021import java.util.HashMap;
022import java.util.LinkedList;
023import java.util.List;
024import java.util.Map;
025import java.util.Objects;
026
027import org.eclipse.january.DatasetException;
028import org.eclipse.january.IMonitor;
029import org.eclipse.january.io.ILazyLoader;
030import org.eclipse.january.metadata.MetadataFactory;
031import org.eclipse.january.metadata.MetadataType;
032import org.eclipse.january.metadata.OriginMetadata;
033import org.eclipse.january.metadata.Reshapeable;
034import org.eclipse.january.metadata.Sliceable;
035import org.eclipse.january.metadata.Transposable;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable {
040        private static final long serialVersionUID = 2467865859867440242L;
041
042        private static final Logger logger = LoggerFactory.getLogger(LazyDataset.class);
043
044        protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null;
045        protected int[] oShape; // original shape
046        protected long  size;   // number of items
047        private Class<? extends Dataset> clazz = null;
048        protected int   isize;  // number of elements per item
049
050        protected ILazyLoader loader;
051
052        // relative to loader
053        protected int[] begSlice = null; // slice begin
054        protected int[] delSlice = null; // slice delta
055        /**
056         * @since 2.2
057         */
058        protected int[] sShape   = null; // sliced shape
059
060        /**
061         * @since 2.2
062         */
063        protected int[] padding = null; // differences in shape from original (or sliced) shape
064        protected int[] map; // transposition map (same length as current shape)
065
066        /**
067         * Create a lazy dataset
068         * @param loader lazy loader
069         * @param name of dataset
070         * @param elements item size
071         * @param clazz dataset sub-interface
072         * @param shape dataset shape
073         * @since 2.3
074         */
075        public LazyDataset(ILazyLoader loader, String name, int elements, Class<? extends Dataset> clazz, int... shape) {
076                this.loader = loader;
077                this.name = name;
078                this.isize = elements;
079                this.clazz = clazz;
080                this.shape = shape.clone();
081                this.oShape = this.shape;
082                try {
083                        size = ShapeUtils.calcLongSize(shape);
084                } catch (IllegalArgumentException e) {
085                        size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 
086                }
087        }
088
089        /**
090         * Create a lazy dataset
091         * @param loader lazy loader
092         * @param name of dataset
093         * @param clazz dataset sub-interface
094         * @param shape dataset shape
095         * @since 2.3
096         */
097        public LazyDataset(ILazyLoader loader, String name, Class<? extends Dataset> clazz, int... shape) {
098                this(loader, name, 1, clazz, shape);
099        }
100
101        /**
102         * Create a lazy dataset
103         * @param name of dataset
104         * @param dtype dataset type
105         * @param elements item size
106         * @param shape dataset shape
107         * @param loader lazy loader
108         * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])}
109         */
110        @Deprecated
111        public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) {
112                this(loader, name, elements, DTypeUtils.getInterface(dtype), shape);
113        }
114
115        /**
116         * Create a lazy dataset
117         * @param name of dataset
118         * @param dtype dataset type
119         * @param shape dataset shape
120         * @param loader lazy loader
121         * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])}
122         */
123        @Deprecated
124        public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) {
125                this(name, dtype, 1, shape, loader);
126        }
127
128        LazyDataset(LazyDataset other) {
129                name  = other.name;
130                shape = other.shape.clone();
131                metadata  = other.copyMetadata();
132                oMetadata = other.oMetadata;
133                oShape = other.oShape;
134                size   = other.size;
135                clazz  = other.clazz;
136                isize  = other.isize;
137                loader = other.loader;
138
139                begSlice = other.begSlice;
140                delSlice = other.delSlice;
141                sShape   = other.sShape;
142                padding  = other.padding;
143                map      = other.map;
144        }
145
146        /**
147         * Create a lazy dataset based on in-memory data (handy for testing)
148         * @param dataset input
149         * @return lazy dataset
150         */
151        public static LazyDataset createLazyDataset(final Dataset dataset) {
152                return new LazyDataset(new ILazyLoader() {
153                        private static final long serialVersionUID = -6725268922780517523L;
154
155                        final Dataset d = dataset;
156
157                        @Override
158                        public boolean isFileReadable() {
159                                return true;
160                        }
161
162                        @Override
163                        public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException {
164                                return d.getSlice(mon, slice);
165                        }
166                }, dataset.getName(), dataset.getElementsPerItem(), dataset.getClass(), dataset.getShapeRef());
167        }
168
169        @Override
170        public Class<?> getElementClass() {
171                return InterfaceUtils.getElementClass(clazz);
172        }
173
174        /**
175         * Can return -1 when dataset interface is not defined
176         */
177        @Override
178        public int getDType() {
179                return clazz == null ? -1 : DTypeUtils.getDType(clazz);
180        }
181
182        /**
183         * @return dataset interface that supports element class and number of elements. Can be null when undefined
184         * @since 2.3
185         */
186        public Class<? extends Dataset> getInterface() {
187                return clazz;
188        }
189
190        /**
191         * Set interface
192         * @param clazz dataset sub-interface
193         * @since 2.3
194         */
195        public void setInterface(Class<? extends Dataset> clazz) {
196                this.clazz = clazz;
197        }
198
199        /**
200         * Can return -1 for unknown
201         */
202        @Override
203        public int getElementsPerItem() {
204                return isize;
205        }
206
207        @Override
208        public int getSize() {
209                return (int) size;
210        }
211
212        @Override
213        public String toString() {
214                StringBuilder out = new StringBuilder();
215
216                if (name != null && name.length() > 0) {
217                        out.append("Lazy dataset '");
218                        out.append(name);
219                        out.append("' has shape [");
220                } else {
221                        out.append("Lazy dataset shape is [");
222                }
223                int rank = shape == null ? 0 : shape.length;
224
225                if (rank > 0 && shape[0] >= 0) {
226                        out.append(shape[0]);
227                }
228                for (int i = 1; i < rank; i++) {
229                        out.append(", " + shape[i]);
230                }
231                out.append(']');
232
233                return out.toString();
234        }
235
236        @Override
237        public int hashCode() {
238                final int prime = 31;
239                int result = super.hashCode();
240                result = prime * result + Arrays.hashCode(oShape);
241                result = prime * result + (int) (size ^ (size >>> 32));
242                result = prime * result + Objects.hashCode(clazz);
243                result = prime * result + isize;
244                result = prime * result + Objects.hashCode(loader);
245                result = prime * result + Arrays.hashCode(begSlice);
246                result = prime * result + Arrays.hashCode(delSlice);
247                result = prime * result + Arrays.hashCode(sShape);
248                result = prime * result + Arrays.hashCode(padding);
249                result = prime * result + Arrays.hashCode(map);
250                return result;
251        }
252
253        @Override
254        public boolean equals(Object obj) {
255                if (this == obj) {
256                        return true;
257                }
258                if (!super.equals(obj)) {
259                        return false;
260                }
261
262                LazyDataset other = (LazyDataset) obj;
263                if (!Arrays.equals(oShape, other.oShape)) {
264                        return false;
265                }
266                if (size != other.size) {
267                        return false;
268                }
269                if (!Objects.equals(clazz, other.clazz)) {
270                        return false;
271                }
272                if (isize != other.isize) {
273                        return false;
274                }
275
276                if (loader != other.loader) {
277                        return false;
278                }
279
280                if (!Arrays.equals(begSlice, other.begSlice)) {
281                        return false;
282                }
283                if (!Arrays.equals(delSlice, other.delSlice)) {
284                        return false;
285                }
286                if (!Arrays.equals(sShape, other.sShape)) {
287                        return false;
288                }
289                if (!Arrays.equals(padding, other.padding)) {
290                        return false;
291                }
292                if (!Arrays.equals(map, other.map)) {
293                        return false;
294                }
295
296                return true;
297        }
298
299        @Override
300        public LazyDataset clone() {
301                return new LazyDataset(this);
302        }
303
304        @Override
305        public void setShape(int... shape) {
306                setShapeInternal(shape.clone());
307        }
308
309        @Override
310        public LazyDataset squeezeEnds() {
311                setShapeInternal(ShapeUtils.squeezeShape(shape, true));
312                return this;
313        }
314
315        @Override
316        public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException {
317                return getSlice(null, start, stop, step);
318        }
319
320        @Override
321        public Dataset getSlice(Slice... slice) throws DatasetException {
322                if (slice == null || slice.length == 0) {
323                        return internalGetSlice(null, new SliceND(shape));
324                }
325                return internalGetSlice(null, new SliceND(shape, slice));
326        }
327
328        @Override
329        public Dataset getSlice(SliceND slice) throws DatasetException {
330                return getSlice(null, slice);
331        }
332
333        @Override
334        public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException {
335                if (slice == null || slice.length == 0) {
336                        return internalGetSlice(monitor, new SliceND(shape));
337                }
338                return internalGetSlice(monitor, new SliceND(shape, slice));
339        }
340
341        @Override
342        public LazyDataset getSliceView(Slice... slice) {
343                if (slice == null || slice.length == 0) {
344                        return internalGetSliceView(new SliceND(shape));
345                }
346                return internalGetSliceView(new SliceND(shape, slice));
347        }
348
349        /**
350         * @param nShape
351         */
352        private void setShapeInternal(int... nShape) {
353                // work out transposed (sliced) shape (instead of removing padding from current shape)
354                if (size != 0) {
355                        int[] pShape = calcTransposed(map, sShape == null ? oShape : sShape);
356                        padding = ShapeUtils.calcShapePadding(pShape, nShape);
357                }
358
359                if (metadata != null) {
360                        storeMetadata(metadata, Reshapeable.class);
361                        metadata = copyMetadata();
362                        reshapeMetadata(shape, nShape);
363                }
364                shape = nShape;
365        }
366
367        @Override
368        public LazyDataset getSliceView(int[] start, int[] stop, int[] step) {
369                return internalGetSliceView(new SliceND(shape, start, stop, step));
370        }
371
372        @Override
373        public LazyDataset getSliceView(SliceND slice) {
374                if (slice != null) {
375                        checkSliceND(slice);
376                }
377                return internalGetSliceView(slice);
378        }
379
380        protected LazyDataset internalGetSliceView(SliceND slice) {
381                LazyDataset view = clone();
382                if (slice == null || slice.isAll()) {
383                        return view;
384                }
385
386                SliceND nslice = calcTrueSlice(slice);
387                if (nslice != null) {
388                        view.begSlice = nslice.getStart();
389                        view.delSlice = nslice.getStep();
390                        view.sShape = nslice.getShape();
391                }
392                view.shape = slice.getShape();
393                view.size = ShapeUtils.calcLongSize(view.shape);
394                view.storeMetadata(metadata, Sliceable.class);
395
396                view.sliceMetadata(true, slice);
397                return view;
398        }
399
400        @Override
401        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException {
402                return internalGetSlice(monitor, new SliceND(shape, start, stop, step));
403        }
404
405        @Override
406        public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException {
407                if (slice != null) {
408                        checkSliceND(slice);
409                }
410                return internalGetSlice(monitor, slice);
411        }
412
413        protected Dataset internalGetSlice(IMonitor monitor, SliceND slice) throws DatasetException {
414                if (loader != null && !loader.isFileReadable()) {
415                        return null;
416                }
417
418                SliceND nslice = calcTrueSlice(slice);
419
420                Dataset a;
421                if (nslice == null) {
422                        a = DatasetFactory.zeros(clazz == null ? DoubleDataset.class : clazz, slice == null ? shape : slice.getShape());
423                } else {
424                        try {
425                                a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice));
426                        } catch (IOException e) {
427                                logger.error("Problem getting {}: {}", slice == null ? "all" : String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()),
428                                                                Arrays.toString(slice.getStep()), loader), e);
429                                throw new DatasetException(e);
430                        }
431                }
432                a.setName(name + AbstractDataset.BLOCK_OPEN + (nslice == null ?
433                                (slice == null ? AbstractDataset.ELLIPSIS : slice) : nslice) + AbstractDataset.BLOCK_CLOSE);
434                if (metadata != null && a instanceof LazyDatasetBase) {
435                        LazyDatasetBase ba = (LazyDatasetBase) a;
436                        ba.metadata = copyMetadata();
437                        if (oMetadata != null) {
438                                ba.restoreMetadata(oMetadata);
439                        }
440                        // metadata axis may be larger than data
441                        if (nslice != null && (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape())) {
442                                ba.sliceMetadata(true, nslice);
443                        }
444                }
445
446                if (nslice != null) {
447                        if (map != null) {
448                                a = a.getTransposedView(map);
449                        }
450                        if (padding != null) {
451                                a.setShape(slice == null ? shape : slice.getShape());
452                        }
453                }
454                a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice == null ?
455                                (slice == null ? null : slice.convertToSlice()) : nslice.convertToSlice(), oShape, null, name));
456
457                if (clazz == null) {
458                        clazz = a.getClass();
459                }
460                return a;
461        }
462
463        @Override
464        public LazyDataset getTransposedView(final int... axes) {
465                LazyDataset view = clone();
466
467                int[] naxes = checkPermutatedAxes(shape, axes);
468                if (naxes == null) {
469                        return view;
470                }
471
472                view.shape = calcTransposed(naxes, shape);
473                if (view.size != 0 && padding != null) { // work out transpose by reverting effect of padding
474                        int or = oShape.length;
475                        int nr = shape.length;
476                        int j = 0; // naxes index
477                        int[] mShape = calcTransposed(map, sShape == null ? oShape : sShape); // pre-padded shape
478                        int m = 0; // shape index
479                        int e = -1; // index of unit dimension
480                        final List<Integer> uaxes = new LinkedList<>();
481                        for (int a : naxes) {
482                                uaxes.add(a);
483                        }
484                        List<Integer> oList = new ArrayList<>(); // dimensions left out by padding (in order)
485                        int np = padding.length;
486                        for (int i = 0; i < np; i++) {
487                                int p = padding[i];
488                                if (p > 0) { // remove added dimensions
489                                        for (int k = 0; k < p; k++, j++) {
490                                                uaxes.remove((Integer) j);
491                                        }
492                                } else if (p == 0) { // leave alone
493                                        if (mShape[m] == 1) { // bump up last unit dimension index
494                                                e = m;
495                                        }
496                                        j++;
497                                        m++;
498                                } else { // add omitted dimensions to list
499                                        p = -p;
500                                        for (int k = 0; k < p; k++) {
501                                                e = find(mShape, 1, e + 1);
502                                                oList.add(e);
503                                        }
504                                }
505                        }
506                        
507                        int[] omitted = new int[oList.size()];
508                        j = 0;
509                        for (Integer o : oList) {
510                                omitted[j++] = o;
511                        }
512                        int[] used = new int[or - omitted.length]; // all dimensions not omitted in pre-padded shape
513                        j = 0;
514                        for (int i = 0; i < or; i++) {
515                                if (Arrays.binarySearch(omitted, i) < 0) {
516                                        used[j++] = i;
517                                }
518                        }
519
520                        int[] vaxes = new int[uaxes.size()];
521                        j = 0;
522                        for (int i = 0; i < nr; i++) { // remap dimension numbering
523                                int l = uaxes.indexOf(i);
524                                if (l >= 0) {
525                                        vaxes[l] = used[j++];
526                                }
527                        }
528                        int[] taxes = new int[or];
529                        j = 0;
530                        for (int i = 0; i < or; i++) { // reassemble map
531                                if (Arrays.binarySearch(omitted, i) >= 0) {
532                                        taxes[i] = i;
533                                } else {
534                                        taxes[i] = vaxes[j++];
535                                }
536                        }
537
538                        naxes = taxes;
539                }
540
541                view.map = map == null ? naxes : calcTransposed(naxes, map);
542                if (view.size != 0) {
543                        // work out transposed (sliced) shape
544                        int[] tShape = calcTransposed(view.map, sShape == null ? oShape : sShape);
545                        try {
546                                view.padding = ShapeUtils.calcShapePadding(tShape, view.shape);
547                        } catch (IllegalArgumentException e) {
548                                System.err.println(e.getMessage() + ": " + Arrays.toString(tShape) + " cf " + Arrays.toString(view.shape));
549                        }
550                }
551                view.storeMetadata(metadata, Transposable.class);
552                view.transposeMetadata(axes);
553                return view;
554        }
555
556        private static int find(int[] map, int m, int off) {
557                for (int i = off, imax = map.length; i < imax; i++) {
558                        if (map[i] == m) {
559                                return i;
560                        }
561                }
562                return -1;
563        }
564
565        private static int[] calcTransposed(int[] map, int[] values) {
566                if (values == null) {
567                        return null;
568                }
569                int r = values.length;
570                if (map == null || r < 2) {
571                        return values;
572                }
573                int[] ovalues = new int[r];
574                for (int i = 0; i < r; i++) {
575                        ovalues[i] = values[map[i]];
576                }
577                return ovalues;
578        }
579
580        /**
581         * Calculate absolute slice
582         * @param slice an n-D slice
583         * @return true slice or null if zero-sized
584         */
585        protected final SliceND calcTrueSlice(SliceND slice) {
586                /*
587                 * Lazy dataset operations: getTransposedView (T), getSliceView (G), setShape/squeezeEnds (S+/S-):
588                 * 
589                 *     . T sets shape, base, and map in new view
590                 *     . G sets shape, size, begSlice and delSlice in new view
591                 *     . S sets shape, shapePadding in current view
592                 * 
593                 * Then getSlice needs to interpret all info to find true slice, load data, get transposition (view)
594                 * and set shape. Therefore:
595                 *     . S needs to update shapePadding only
596                 *     . T needs to update shapePadding too
597                 *     . G needs to work out true slice to update
598                 * 
599                 * slice -> true slice
600                 *   adjusts for shape (S^-1) then remap dimensions (T^-1)
601                 */
602
603                if (slice == null) {
604                        slice = new SliceND(shape);
605                }
606
607                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
608                        return null;
609                }
610
611                int[] nshape;
612                int[] nstart;
613                int[] nstep;
614
615                int r = oShape.length;
616                if (padding == null) {
617                        nshape = slice.getShape();
618                        nstart = slice.getStart();
619                        nstep = slice.getStep();
620                } else {
621                        final int[] lshape = slice.getShape();
622                        final int[] lstart = slice.getStart();
623                        final int[] lstep  = slice.getStep();
624
625                        nstart = new int[r];
626                        nstep = new int[r];
627                        nshape = new int[r];
628                        int i = 0;
629                        int j = 0;
630                        for (int p : padding) { // remove padding
631                                if (p == 0) {
632                                        nshape[i] = lshape[j];
633                                        nstart[i] = lstart[j];
634                                        nstep[i]  = lstep[j];
635                                        i++;
636                                        j++;
637                                } else if (p < 0) {
638                                        int imax = i - p;
639                                        while (i < imax) {
640                                                nshape[i] = 1;
641                                                nstep[i]  = 1;
642                                                i++;
643                                        }
644                                } else {
645                                        j += p;
646                                }
647                        }
648                }
649
650                if (map != null && r > 1) { // transpose dimensions
651                        int[] pshape = new int[r];
652                        int[] pstart = new int[r];
653                        int[] pstep = new int[r];
654                        for (int i = 0; i < r; i++) {
655                                int m = map[i];
656                                pshape[m] = nshape[i];
657                                pstart[m] = nstart[i];
658                                pstep[m]  = nstep[i];
659                        }
660
661                        nshape = pshape;
662                        nstart = pstart;
663                        nstep  = pstep;
664                }
665
666                int[] nstop = new int[r];
667                if (begSlice != null) { // find net slice
668                        for (int i = 0; i < r; i++) {
669                                int b = begSlice[i];
670                                int d = delSlice[i];
671                                nstart[i] = b + nstart[i] * d;
672                                int nd = nstep[i] * d;
673                                nstep[i] = nd;
674                                nstop[i]  = nstart[i] + (nshape[i] - 1) * nd + (nd >= 0 ? 1 : -1);
675                        }
676                } else {
677                        for (int i = 0; i < r; i++) {
678                                int d = nstep[i];
679                                nstop[i] = nstart[i] + (nshape[i] - 1) * d + (d >= 0 ? 1 : -1);
680                        }
681                }
682
683                return createSlice(nstart, nstop, nstep);
684        }
685
686        protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) {
687                return SliceND.createSlice(oShape, null, nstart, nstop, nstep);
688        }
689
690        /**
691         * Transform data so that it can be used in setSlice of saver
692         * @param data input
693         * @param tslice true slice 
694         * @return data with dimensions adjusted and remapped 
695         */
696        final IDataset transformInput(IDataset data, SliceND tslice) {
697                if (padding != null) { // remove padding
698                        data = data.getSliceView();
699                        int[] nshape = tslice == null ? shape : tslice.getShape();
700                        data.setShape(nshape);
701                }
702
703                return map == null ? data : data.getTransposedView(map);
704        }
705
706        /**
707         * Store metadata items that has given annotation
708         * @param origMetadata original metadata
709         * @param aclazz annotation class
710         */
711        private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) {
712                List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz);
713                if (mclazzes.size() == 0) {
714                        return;
715                }
716
717                if (oMetadata == null) {
718                        oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>();
719                }
720                for (Class<? extends MetadataType> mc : mclazzes) {
721                        if (oMetadata.containsKey(mc)) {
722                                continue; // do not overwrite original
723                        }
724
725                        List<MetadataType> l = origMetadata.get(mc);
726                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
727                        for (MetadataType m : l) {
728                                nl.add(m.clone());
729                        }
730                        oMetadata.put(mc, nl);
731                }
732        }
733
734        @SuppressWarnings("unchecked")
735        private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) {
736                List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>();
737                if (metadata == null) {
738                        return mclazzes;
739                }
740
741                for (Class<? extends MetadataType> c : metadata.keySet()) {
742                        boolean hasAnn = false;
743                        for (MetadataType m : metadata.get(c)) {
744                                if (m == null) {
745                                        continue;
746                                }
747
748                                Class<? extends MetadataType> mc = m.getClass();
749                                do { // iterate over super-classes
750                                        for (Field f : mc.getDeclaredFields()) {
751                                                if (f.isAnnotationPresent(aclazz)) {
752                                                        hasAnn = true;
753                                                        break;
754                                                }
755                                        }
756                                        Class<?> sclazz = mc.getSuperclass();
757                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
758                                                break;
759                                        }
760                                        mc = (Class<? extends MetadataType>) sclazz;
761                                } while (!hasAnn);
762                                if (hasAnn) {
763                                        break;
764                                }
765                        }
766                        if (hasAnn) {
767                                mclazzes.add(c);
768                        }
769                }
770                return mclazzes;
771        }
772
773        /**
774         * Gets the maximum size of a slice of a dataset in a given dimension
775         * which should normally fit in memory. Note that it might be possible
776         * to get more in memory, this is a conservative estimate and seems to
777         * almost always work at the size returned; providing Xmx is less than
778         * the physical memory.
779         * 
780         * To get more in memory increase -Xmx setting or use an expression
781         * which calls a rolling function (like rmean) instead of slicing directly
782         * to memory.
783         * 
784         * @param lazySet lazy dataset
785         * @param dimension to slice along
786         * @return maximum size of dimension that can be sliced.
787         */
788        public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) {
789                // size in bytes of each item
790                final double size = InterfaceUtils.getItemBytes(lazySet.getElementsPerItem(), InterfaceUtils.getInterface(lazySet));
791                
792                // Max in bytes takes into account our minimum requirement
793                final double max  = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory());
794                
795                // Firstly if the whole dataset it likely to fit in memory, then we allow it.
796                // Space specified in bytes per item available
797                final double space = max/lazySet.getSize();
798
799                // If we have room for this whole dataset, then fine
800                int[] shape = lazySet.getShape();
801                if (space >= size) {
802                        return shape[dimension];
803                }
804
805                // Otherwise estimate what we can fit in, conservatively.
806                // First get size of one slice, see it that fits, if not, still return 1
807                double sizeOneSlice = size; // in bytes
808                for (int dim = 0; dim < shape.length; dim++) {
809                        if (dim == dimension) {
810                                continue;
811                        }
812                        sizeOneSlice *= shape[dim];
813                }
814                double avail = max / sizeOneSlice;
815                if (avail < 1) {
816                        return 1;
817                }
818
819                // We fudge this to leave some room
820                return (int) Math.floor(avail/4d);
821        }
822}