[S#3543] Piece-wise linear compression of column groups first working prototype #2415 #2420

janniklinde · 2026-01-30T10:07:21Z

This file should not be part of the PR. You can keep it locally but you should untrack it and not add it to your commits. You could use git rm --cached bin/systemds-standalone.sh.

janniklinde · 2026-01-30T10:09:57Z

It seems like you reformatted the file to revert the tabs -> spaces conversion, which is good. However, there are still many unnecessary changes. I would recommend you revert that file to the original state of this repository and then only add the enum CompressionType PiecewiseLinear

-Original file line number
+Diff line change
@@ -0,0 +1,12 @@
+    #!/bin/bash
+    # Standalone-Launcher für SystemDS
+    SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+    JAR_FILE="$SCRIPT_DIR/../target/systemds-3.4.0-SNAPSHOT.jar"
+    if [ ! -f "$JAR_FILE" ]; then
+      echo "ERROR: Standalone JAR nicht gefunden: $JAR_FILE"
+      exit 1
+    fi
+    java -cp "$JAR_FILE" org.apache.sysds.api.DMLScript "$@"

-Original file line number
+Diff line change
@@ Expand Up / @@ -55,16 +55,16 @@ public class CompressionSettings { @@
     	/**
     	 * The sampling ratio used when choosing ColGroups. Note that, default behavior is to use exact estimator if the
     	 * number of elements is below 1000.
-    	 *
+    	 *
     	 * DEPRECATED
     	 */
     	public final double samplingRatio;
     	/**
     	 * The sampling ratio power to use when choosing sample size. This is used in accordance to the function:
-    	 *
+    	 *
     	 * sampleSize += nRows^samplePower;
-    	 *
+    	 *
     	 * The value is bounded to be in the range of 0 to 1, 1 giving a sample size of everything, and 0 adding 1.
     	 */
     	public final double samplePower;
@@ Expand Down Expand Up / @@ -114,8 +114,9 @@ public class CompressionSettings { @@
     	/**
     	 * Transpose input matrix, to optimize access when extracting bitmaps. This setting is changed inside the script
     	 * based on the transposeInput setting.
-    	 *
-    	 * This is intentionally left as a mutable value, since the transposition of the input matrix is decided in phase 3.
+    	 *
+    	 * This is intentionally left as a mutable value, since the transposition of the input matrix is decided in phase
+    	 * 3.
     	 */
     	public boolean transposed = false;
@@ Expand All / @@ -135,6 +136,19 @@ public class CompressionSettings { @@
     	public final boolean preferDeltaEncoding;
+    	// Handling Targetloss for piecewise linear Kompression
+    	private double piecewiseTargetLoss = Double.NaN;
+    	public void setPiecewiseTargetLoss(double piecewiseTargetLoss) {
+    		this.piecewiseTargetLoss = piecewiseTargetLoss;
+    	}
+    	public double getPiecewiseTargetLoss() {
+    		return piecewiseTargetLoss;
+    	}
     	protected CompressionSettings(double samplingRatio, double samplePower, boolean allowSharedDictionary,
     		String transposeInput, int seed, boolean lossy, EnumSet<CompressionType> validCompressions,
     		boolean sortValuesByLength, PartitionerType columnPartitioner, int maxColGroupCoCode, double coCodePercentage,
@@ Expand All @@
     		this.sdcSortType = sdcSortType;
     		this.scaleFactors = scaleFactors;
     		this.preferDeltaEncoding = preferDeltaEncoding;
     		if(!printedStatus && LOG.isDebugEnabled()) {
     			printedStatus = true;
     			LOG.debug(this.toString());
@@ Expand Down @@

-Original file line number
+Diff line change
@@ Expand Up / @@ -65,7 +65,7 @@ public abstract class AColGroup implements Serializable { @@
     	/** Public super types of compression ColGroups supported */
     	public static enum CompressionType {
-    		UNCOMPRESSED, RLE, OLE, DDC, CONST, EMPTY, SDC, SDCFOR, DDCFOR, DeltaDDC, LinearFunctional;
+    		UNCOMPRESSED, RLE, OLE, DDC, CONST, EMPTY, SDC, SDCFOR, DDCFOR, DeltaDDC, LinearFunctional, PiecewiseLinear;
     		public boolean isDense() {
     			return this == DDC || this == CONST || this == DDCFOR || this == DDCFOR;
@@ Expand Down @@

-Original file line number
+Diff line change
@@ Expand Up / @@ -43,6 +43,7 @@ @@
     import org.apache.sysds.runtime.compress.colgroup.dictionary.DictionaryFactory;
     import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary;
     import org.apache.sysds.runtime.compress.colgroup.functional.LinearRegression;
+    import org.apache.sysds.runtime.compress.colgroup.functional.PiecewiseLinearUtils;
     import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
     import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
     import org.apache.sysds.runtime.compress.colgroup.insertionsort.AInsertionSorter;
@@ Expand Down Expand Up @@
     	/**
     	 * The actual compression method, that handles the logic of compressing multiple columns together.
-    	 *
+    	 *
     	 * @param in  The input matrix, that could have been transposed. If it is transposed the compSettings should specify
     	 *            this.
     	 * @param csi The compression information extracted from the estimation, this contains which groups of columns to
@@ Expand All @@
     	/**
     	 * The actual compression method, that handles the logic of compressing multiple columns together.
-    	 *
+    	 *
     	 * @param in  The input matrix, that could have been transposed. If it is transposed the compSettings should specify
     	 *            this.
     	 * @param csi The compression information extracted from the estimation, this contains which groups of columns to
@@ Expand All @@
     	}
     	/**
-    	 *
+    	 *
     	 * @param in  The input matrix, that could have been transposed. If it is transposed the compSettings should specify
     	 *            this.
     	 * @param csi The compression information extracted from the estimation, this contains which groups of columns to
@@ Expand Down Expand Up @@
     					time, retType, estC, actC, act.getNumValues(), cols, wanted, warning));
     		}
     		else {
-    			LOG.debug(String.format("time[ms]: %10.2f %25s est %10.0f -- act %10.0f distinct:%5d cols:%s wanted:%s",
-    				time, retType, estC, actC, act.getNumValues(), cols, wanted));
+    			LOG.debug(
+    				String.format("time[ms]: %10.2f %25s est %10.0f -- act %10.0f distinct:%5d cols:%s wanted:%s", time,
+    					retType, estC, actC, act.getNumValues(), cols, wanted));
     		}
     	}
@@ Expand Down Expand Up / @@ -303,6 +305,9 @@ else if(ct == CompressionType.LinearFunctional) { @@
     				return compressLinearFunctional(colIndexes, in, cs);
     			}
     		}
+    		else if(ct == CompressionType.PiecewiseLinear) {
+    			return compressPiecewiseLinearFunctional(colIndexes, in, cs);
+    		}
     		else if(ct == CompressionType.DDCFOR) {
     			AColGroup g = directCompressDDC(colIndexes, cg);
     			if(g instanceof ColGroupDDC)
@@ Expand Down Expand Up @@
     		if(cs.scaleFactors != null) {
     			throw new NotImplementedException("Delta encoding with quantization not yet implemented");
     		}
     		if(colIndexes.size() > 1) {
     			return directCompressDeltaDDCMultiCol(colIndexes, cg);
     		}
@@ Expand Down Expand Up @@
     		if(map.size() == 0)
     			return new ColGroupEmpty(colIndexes);
     		final double[] dictValues = map.getDictionary();
     		IDictionary dict = new DeltaDictionary(dictValues, 1);
@@ Expand All @@
     		return ColGroupDeltaDDC.create(colIndexes, dict, resData, null);
     	}
-    	private AColGroup directCompressDeltaDDCMultiCol(IColIndex colIndexes, CompressedSizeInfoColGroup cg) throws Exception {
+    	private AColGroup directCompressDeltaDDCMultiCol(IColIndex colIndexes, CompressedSizeInfoColGroup cg)
+    		throws Exception {
     		final AMapToData d = MapToFactory.create(nRow, Math.max(Math.min(cg.getNumOffs() + 1, nRow), 126));
     		final int fill = d.getUpperBoundValue();
     		d.fill(fill);
@@ Expand Down Expand Up @@
     		int fill) {
     		ReaderColumnSelection reader = (cs.scaleFactors == null) ? ReaderColumnSelection.createReader(in, colIndexes,
-    			cs.transposed, rl,
-    			ru) : ReaderColumnSelection.createQuantizedReader(in, colIndexes, cs.transposed, rl, ru, cs.scaleFactors);
+    			cs.transposed, rl, ru) : ReaderColumnSelection.createQuantizedReader(in, colIndexes, cs.transposed, rl, ru,
+    			cs.scaleFactors);
     		DblArray cellVals = reader.nextRow();
     		boolean extra = false;
@@ Expand Down Expand Up @@
     		return ColGroupLinearFunctional.create(colIndexes, coefficients, numRows);
     	}
+    	public static AColGroup compressPiecewiseLinearFunctional(
+    		IColIndex colIndexes, MatrixBlock in, CompressionSettings cs) {
+    		final int numRows = in.getNumRows();
+    		AColGroup result = null;
+    		//Compress every column
+    		for (int col = 0; col < colIndexes.size(); col++) {
+    			// get Column Index
+    			IColIndex.SliceResult sliceResult = colIndexes.slice(col, col + 1);
+    			IColIndex singleColIndex = sliceResult.ret;  // ← .ret nötig!
+    			// Get Column from Matrix
+    			final int colIdx = colIndexes.get(col);
+    			double[] column = PiecewiseLinearUtils.getColumn(in, colIdx);
+    			//Compress column
+    			PiecewiseLinearUtils.SegmentedRegression fit =
+    				PiecewiseLinearUtils.compressSegmentedLeastSquares(column, cs);
+    			AColGroup singleGroup = ColGroupPiecewiseLinearCompressed.create(
+    				singleColIndex,
+    				fit.getBreakpoints(),
+    				fit.getSlopes(),
+    				fit.getIntercepts(),
+    				numRows);
+    			// Combine multiple columns
+    			if (result == null) {
+    				result = singleGroup;
+    			} else {
+    				result = result.combineWithSameIndex(numRows, col, singleGroup);
+    			}
+    		}
+    		return result;
+    	}
     	private AColGroup compressSDCFromSparseTransposedBlock(IColIndex cols, int nrUniqueEstimate, double tupleSparsity) {
     		if(cols.size() > 1)
     			return compressMultiColSDCFromSparseTransposedBlock(cols, nrUniqueEstimate, tupleSparsity);
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[S#3543] Piece-wise linear compression of column groups first working prototype #2415 #2420

Uh oh!

Diff view

Diff view

There are no files selected for viewing

janniklinde Jan 30, 2026

Uh oh!

janniklinde Jan 30, 2026

Uh oh!

Uh oh!

[S#3543] Piece-wise linear compression of column groups first working prototype #2415 #2420

Are you sure you want to change the base?

Uh oh!

[S#3543] Piece-wise linear compression of column groups first working prototype #2415 #2420

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

janniklinde Jan 30, 2026

Choose a reason for hiding this comment

Uh oh!

janniklinde Jan 30, 2026

Choose a reason for hiding this comment

Uh oh!

Uh oh!