[SYSTEMDS-3644] Compressed transform encode

src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java

@@ -348,6 +348,7 @@ private void classifyPhase() {
 				LOG.info("Threshold was set to : " + threshold + " but it was above original " + _stats.originalCost);
 				LOG.info("Original size       : " + _stats.originalSize);
 				LOG.info("single col size     : " + _stats.estimatedSizeCols);
+				LOG.debug(String.format("--compressed size:   %16d", _stats.originalSize));
 				if(!(costEstimator instanceof MemoryCostEstimator)) {
 					LOG.info("original cost      : " + _stats.originalCost);
 					LOG.info("single col cost    : " + _stats.estimatedCostCols);

src/main/java/org/apache/sysds/runtime/compress/cocode/CoCodeGreedy.java

@@ -224,7 +224,8 @@ public Object call() throws Exception {
 				final int maxCombined = c1i.getNumVals() * c2i.getNumVals();
 
 				if(maxCombined < 0 // int overflow
-					|| maxCombined > c1i.getNumRows()) // higher combined than number of rows.
+					|| maxCombined > c1i.getNumRows() // higher than number of rows
+					|| maxCombined > 100000) // higher than 100k ... then lets not precalculate it.
 					return null;
 
 				final IColIndex c = _c1._indexes.combine(_c2._indexes);

src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java

@@ -94,7 +94,9 @@ public static long getExactSizeOnDisk(List<AColGroup> colGroups) {
 			}
 			ret += grp.getExactSizeOnDisk();
 		}
-		LOG.error(" duplicate dicts on exact Size on Disk : " + (colGroups.size() - dicts.size()) );
+		if(LOG.isWarnEnabled())
+			LOG.warn(" duplicate dicts on exact Size on Disk : " + (colGroups.size() - dicts.size()) );
+
 		return ret;
 	}
 

src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java

@@ -34,6 +34,7 @@
 import org.apache.sysds.runtime.compress.colgroup.dictionary.DictionaryFactory;
 import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary;
 import org.apache.sysds.runtime.compress.colgroup.dictionary.MatrixBlockDictionary;
+import org.apache.sysds.runtime.compress.colgroup.dictionary.PlaceHolderDict;
 import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
 import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
 import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
@@ -93,7 +94,7 @@ public static AColGroup create(IColIndex colIndices, int numRows, IDictionary di
 		int[] cachedCounts) {
 		if(dict == null)
 			return new ColGroupEmpty(colIndices);
-		else if(data.getUnique() == 1) {
+		else if(data.getUnique() == 1 && !(dict instanceof PlaceHolderDict)) {
 			MatrixBlock mb = dict.getMBDict(colIndices.size()).getMatrixBlock().slice(0, 0);
 			return ColGroupSDCSingleZeros.create(colIndices, numRows, MatrixBlockDictionary.create(mb), offsets, null);
 		}

src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java

@@ -80,7 +80,12 @@ public class ColGroupUncompressed extends AColGroup {
 	 */
 	private final MatrixBlock _data;
 
-	private ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) {
+	/**
+	 * Do not use this constructor of column group uncompressed, instead uce the create constructor.
+	 * @param mb The contained data.
+	 * @param colIndexes Column indexes for this Columngroup
+	 */
+	protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) {
 		super(colIndexes);
 		_data = mb;
 	}

src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java

@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.colgroup;
+
+import org.apache.sysds.runtime.compress.colgroup.ColGroupUtils.P;
+import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
+import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
+import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.data.DenseBlock;
+import org.apache.sysds.runtime.data.SparseBlock;
+import org.apache.sysds.runtime.data.SparseBlockMCSR;
+import org.apache.sysds.runtime.frame.data.columns.Array;
+import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator;
+import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysds.runtime.matrix.operators.CMOperator;
+import org.apache.sysds.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
+
+/**
+ * Special sideways compressed column group not supposed to be used outside of the compressed transform encode.
+ */
+public class ColGroupUncompressedArray extends AColGroup {
+	private static final long serialVersionUID = -825423333043292199L;
+
+	public final Array<?> array;
+	public final int id; // columnID
+
+	public ColGroupUncompressedArray(Array<?> data, int id, IColIndex colIndexes) {
+		super(colIndexes);
+		this.array = data;
+		this.id = id;
+	}
+
+	@Override
+	public int getNumValues() {
+		return array.size();
+	}
+
+	@Override
+	public long estimateInMemorySize() {
+		// not accurate estimate, but guaranteed larger.
+		return MatrixBlock.estimateSizeInMemory(array.size(), 1, array.size()) + 80;
+	}
+
+	@Override
+	public String toString() {
+		return "UncompressedArrayGroup: " + id + " " + _colIndexes;
+	}
+
+	@Override
+	public AColGroup copyAndSet(IColIndex colIndexes) {
+		return new ColGroupUncompressedArray(array, id, colIndexes);	
+	}
+
+	@Override
+	public void decompressToDenseBlockTransposed(DenseBlock db, int rl, int ru) {
+		throw new UnsupportedOperationException("Unimplemented method 'decompressToDenseBlockTransposed'");
+	}
+
+	@Override
+	public void decompressToSparseBlockTransposed(SparseBlockMCSR sb, int nColOut) {
+		throw new UnsupportedOperationException("Unimplemented method 'decompressToSparseBlockTransposed'");
+	}
+
+	@Override
+	public double getIdx(int r, int colIdx) {
+		throw new UnsupportedOperationException("Unimplemented method 'getIdx'");
+	}
+
+	@Override
+	public CompressionType getCompType() {
+		throw new UnsupportedOperationException("Unimplemented method 'getCompType'");
+	}
+
+	@Override
+	protected ColGroupType getColGroupType() {
+		throw new UnsupportedOperationException("Unimplemented method 'getColGroupType'");
+	}
+
+	@Override
+	public void decompressToDenseBlock(DenseBlock db, int rl, int ru, int offR, int offC) {
+		throw new UnsupportedOperationException("Unimplemented method 'decompressToDenseBlock'");
+	}
+
+	@Override
+	public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, int offC) {
+		throw new UnsupportedOperationException("Unimplemented method 'decompressToSparseBlock'");
+	}
+
+	@Override
+	public AColGroup rightMultByMatrix(MatrixBlock right, IColIndex allCols, int k) {
+		throw new UnsupportedOperationException("Unimplemented method 'rightMultByMatrix'");
+	}
+
+	@Override
+	public void tsmm(MatrixBlock ret, int nRows) {
+		throw new UnsupportedOperationException("Unimplemented method 'tsmm'");
+	}
+
+	@Override
+	public void leftMultByMatrixNoPreAgg(MatrixBlock matrix, MatrixBlock result, int rl, int ru, int cl, int cu) {
+		throw new UnsupportedOperationException("Unimplemented method 'leftMultByMatrixNoPreAgg'");
+	}
+
+	@Override
+	public void leftMultByAColGroup(AColGroup lhs, MatrixBlock result, int nRows) {
+		throw new UnsupportedOperationException("Unimplemented method 'leftMultByAColGroup'");
+	}
+
+	@Override
+	public void tsmmAColGroup(AColGroup other, MatrixBlock result) {
+		throw new UnsupportedOperationException("Unimplemented method 'tsmmAColGroup'");
+	}
+
+	@Override
+	public AColGroup scalarOperation(ScalarOperator op) {
+		throw new UnsupportedOperationException("Unimplemented method 'scalarOperation'");
+	}
+
+	@Override
+	public AColGroup binaryRowOpLeft(BinaryOperator op, double[] v, boolean isRowSafe) {
+		throw new UnsupportedOperationException("Unimplemented method 'binaryRowOpLeft'");
+	}
+
+	@Override
+	public AColGroup binaryRowOpRight(BinaryOperator op, double[] v, boolean isRowSafe) {
+		throw new UnsupportedOperationException("Unimplemented method 'binaryRowOpRight'");
+	}
+
+	@Override
+	public void unaryAggregateOperations(AggregateUnaryOperator op, double[] c, int nRows, int rl, int ru) {
+		throw new UnsupportedOperationException("Unimplemented method 'unaryAggregateOperations'");
+	}
+
+	@Override
+	protected AColGroup sliceSingleColumn(int idx) {
+		throw new UnsupportedOperationException("Unimplemented method 'sliceSingleColumn'");
+	}
+
+	@Override
+	protected AColGroup sliceMultiColumns(int idStart, int idEnd, IColIndex outputCols) {
+		throw new UnsupportedOperationException("Unimplemented method 'sliceMultiColumns'");
+	}
+
+	@Override
+	public AColGroup sliceRows(int rl, int ru) {
+		throw new UnsupportedOperationException("Unimplemented method 'sliceRows'");
+	}
+
+	@Override
+	public double getMin() {
+		throw new UnsupportedOperationException("Unimplemented method 'getMin'");
+	}
+
+	@Override
+	public double getMax() {
+		throw new UnsupportedOperationException("Unimplemented method 'getMax'");
+	}
+
+	@Override
+	public double getSum(int nRows) {
+		throw new UnsupportedOperationException("Unimplemented method 'getSum'");
+	}
+
+	@Override
+	public boolean containsValue(double pattern) {
+		throw new UnsupportedOperationException("Unimplemented method 'containsValue'");
+	}
+
+	@Override
+	public long getNumberNonZeros(int nRows) {
+		throw new UnsupportedOperationException("Unimplemented method 'getNumberNonZeros'");
+	}
+
+	@Override
+	public AColGroup replace(double pattern, double replace) {
+		throw new UnsupportedOperationException("Unimplemented method 'replace'");
+	}
+
+	@Override
+	public void computeColSums(double[] c, int nRows) {
+		throw new UnsupportedOperationException("Unimplemented method 'computeColSums'");
+	}
+
+	@Override
+	public CM_COV_Object centralMoment(CMOperator op, int nRows) {
+		throw new UnsupportedOperationException("Unimplemented method 'centralMoment'");
+	}
+
+	@Override
+	public AColGroup rexpandCols(int max, boolean ignore, boolean cast, int nRows) {
+		throw new UnsupportedOperationException("Unimplemented method 'rexpandCols'");
+	}
+
+	@Override
+	public double getCost(ComputationCostEstimator e, int nRows) {
+		throw new UnsupportedOperationException("Unimplemented method 'getCost'");
+	}
+
+	@Override
+	public AColGroup unaryOperation(UnaryOperator op) {
+		throw new UnsupportedOperationException("Unimplemented method 'unaryOperation'");
+	}
+
+	@Override
+	public boolean isEmpty() {
+		throw new UnsupportedOperationException("Unimplemented method 'isEmpty'");
+	}
+
+	@Override
+	public AColGroup append(AColGroup g) {
+		throw new UnsupportedOperationException("Unimplemented method 'append'");
+	}
+
+	@Override
+	protected AColGroup appendNInternal(AColGroup[] groups, int blen, int rlen) {
+		throw new UnsupportedOperationException("Unimplemented method 'appendNInternal'");
+	}
+
+	@Override
+	public ICLAScheme getCompressionScheme() {
+		throw new UnsupportedOperationException("Unimplemented method 'getCompressionScheme'");
+	}
+
+	@Override
+	public AColGroup recompress() {
+		throw new UnsupportedOperationException("Unimplemented method 'recompress'");
+	}
+
+	@Override
+	public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+		throw new UnsupportedOperationException("Unimplemented method 'getCompressionInfo'");
+	}
+
+	@Override
+	protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+		throw new UnsupportedOperationException("Unimplemented method 'fixColIndexes'");
+	}
+
+	@Override
+	public AColGroup reduceCols() {
+		throw new UnsupportedOperationException("Unimplemented method 'reduceCols'");
+	}
+
+	@Override
+	public double getSparsity() {
+		throw new UnsupportedOperationException("Unimplemented method 'getSparsity'");
+	}
+
+	@Override
+	protected void sparseSelection(MatrixBlock selection, P[] points, MatrixBlock ret, int rl, int ru) {
+		throw new UnsupportedOperationException("Unimplemented method 'sparseSelection'");
+	}
+
+	@Override
+	protected void denseSelection(MatrixBlock selection, P[] points, MatrixBlock ret, int rl, int ru) {
+		throw new UnsupportedOperationException("Unimplemented method 'denseSelection'");
+	}
+
+	@Override
+	public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) {
+		throw new UnsupportedOperationException("Unimplemented method 'splitReshape'");
+	}
+
+}

src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ColIndexFactory.java

@@ -126,7 +126,7 @@ else if(contiguous)
 			return ArrayIndex.estimateInMemorySizeStatic(nCol);
 	}
 
-	public static IColIndex combine(List<AColGroup> gs) {
+	public static IColIndex combine(List<? extends AColGroup> gs) {
 		int numCols = 0;
 		for(AColGroup g : gs)
 			numCols += g.getNumCols();

src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java

@@ -166,11 +166,26 @@ public void copyBit(MapToBit d) {
 
 	@Override
 	public int[] getCounts(int[] ret) {
-		for(int i = 0; i < _data.length; i++)
+		final int h = (_data.length) % 8;
+		for(int i = 0; i < h; i++)
 			ret[_data[i] & 0xFF]++;
+		getCountsBy8P(ret, h, _data.length);
 		return ret;
 	}
 
+	private void getCountsBy8P(int[] ret, int s, int e) {
+		for(int i = s; i < e; i += 8) {
+			ret[_data[i] & 0xFF]++;
+			ret[_data[i + 1] & 0xFF]++;
+			ret[_data[i + 2] & 0xFF]++;
+			ret[_data[i + 3] & 0xFF]++;
+			ret[_data[i + 4] & 0xFF]++;
+			ret[_data[i + 5] & 0xFF]++;
+			ret[_data[i + 6] & 0xFF]++;
+			ret[_data[i + 7] & 0xFF]++;
+		}
+	}
+
 	@Override
 	protected void preAggregateDenseToRowBy8(double[] mV, double[] preAV, int cl, int cu, int off) {
 		final int h = (cu - cl) % 8;