Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MINOR] Add DataTypes on memory estimates for HOPS #2221

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[MINOR] Add DataTypes on memory estimates for HOPS #2221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
afee50f
[MINOR] Add DataTypes on memory estimates for HOPS
Baunsgaard Feb 5, 2025
6a8a073
fix infinite recursion
Baunsgaard Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/org/apache/sysds/hops/BinaryOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -562,7 +562,7 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
else //e.g., for append,pow or after inference
sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);

ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
}
return ret;
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/apache/sysds/hops/DataGenOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,11 +219,11 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
else {
//sparsity-aware estimation (dependent on sparse generation approach); for pure dense generation
//we would need to disable sparsity-awareness and estimate via sparsity=1.0
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity);
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity, getDataType());
}
}
else {
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, 1.0);
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, 1.0, getDataType());
}

return ret;
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/apache/sysds/hops/DataOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ else if(dt == DataType.FRAME) {
|| _op == OpOpData.TRANSIENTREAD )
{
double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
ret = OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
}
// output memory estimate is not required for "write" nodes (just input)
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/apache/sysds/hops/IndexingOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
{
// only dense right indexing supported on GPU
double sparsity = isGPUEnabled() ? 1.0 : OptimizerUtils.getSparsity(dim1, dim2, nnz);
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
}

@Override
Expand Down
10 changes: 9 additions & 1 deletion src/main/java/org/apache/sysds/hops/OptimizerUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.sysds.api.DMLScript;
import org.apache.sysds.common.Types.DataType;
import org.apache.sysds.common.Types.ExecMode;
import org.apache.sysds.common.Types.ExecType;
import org.apache.sysds.common.Types.FileFormat;
Expand Down Expand Up @@ -63,8 +64,8 @@
import org.apache.sysds.runtime.meta.MatrixCharacteristics;
import org.apache.sysds.runtime.util.IndexRange;
import org.apache.sysds.runtime.util.UtilFunctions;
import org.apache.sysds.utils.stats.InfrastructureAnalyzer;
import org.apache.sysds.utils.MemoryEstimates;
import org.apache.sysds.utils.stats.InfrastructureAnalyzer;

public class OptimizerUtils
{
Expand Down Expand Up @@ -822,6 +823,13 @@ public static long estimateSizeExactSparsity(long nrows, long ncols, double sp)
return MatrixBlock.estimateSizeInMemory(nrows,ncols,sp);
}

public static long estimateSizeExactSparsity(long nrows, long ncols, double sp, DataType dt){
if(dt == DataType.FRAME)
return estimateSizeExactFrame(nrows, ncols);
else
return estimateSizeExactSparsity(nrows, ncols, sp);
}

/**
* Estimates the footprint (in bytes) for a partitioned in-memory representation of a
* matrix with the given matrix characteristics
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/apache/sysds/hops/ReorgOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ public void computeMemEstimate(MemoTable memo){
protected double computeOutputMemEstimate( long dim1, long dim2, long nnz ) {
//no dedicated mem estimation per op type, because always propagated via refreshSizeInformation
double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/apache/sysds/hops/UnaryOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,7 +366,7 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
} else {
sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
}
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
}

@Override
Expand Down
Loading