Merge pull request #11 from DigitalPebble/ccf_storage

Storage energy estimator based on CCF

Author: jnioche

README.md

@@ -2,6 +2,9 @@
 
 # SPRUCE
 
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+![Build Status](https://github.com/apache/stormcrawler/actions/workflows/maven.yml/badge.svg)
+
 *Spruce* helps estimate the environmental impact of your cloud usage. By leveraging open source models and data, it enriches
 usage reports generated by cloud providers and allows you to build reports and visualisations. Having the greenops and finops data in the same 
 place makes it easier to expose your costs and impacts side by side.

src/main/java/com/digitalpebble/spruce/SparkJob.java

@@ -4,6 +4,7 @@
 
 import com.digitalpebble.spruce.modules.ConstantLoad;
 import com.digitalpebble.spruce.modules.ccf.Networking;
+import com.digitalpebble.spruce.modules.ccf.Storage;
 import com.digitalpebble.spruce.modules.electricitymaps.AverageCarbonIntensity;
 import com.google.common.collect.ImmutableMap;
 import org.apache.spark.sql.Dataset;
@@ -50,7 +51,7 @@ public static void main(String[] args) {
         // compute emissions
 
         final List<EnrichmentModule> modules = List.of(
-                new ConstantLoad(),
+                new Storage(),
                 new Networking(),
                 new AverageCarbonIntensity()
         );

src/main/java/com/digitalpebble/spruce/modules/ccf/Networking.java

@@ -15,7 +15,7 @@
  * Provides an estimate of energy used for networking in and out of data centres.
  * Applies a flat coefficient per Gb
  * @see <a href="https://www.cloudcarbonfootprint.org/docs/methodology#networking">CCF methodology</a>
- * @see <a href="https://github.com/cloud-carbon-footprint/cloud-carbon-footprint/blob/main/packages/aws/src/lib/CostAndUsageTypes.ts#L108">resource file</a>
+ * @see <a href="https://github.com/cloud-carbon-footprint/cloud-carbon-footprint/blob/9f2cf436e5ad020830977e52c3b0a1719d20a8b9/packages/aws/src/lib/CostAndUsageTypes.ts#L25">resource file</a>
  **/
 public class Networking implements EnrichmentModule {
 

src/main/java/com/digitalpebble/spruce/modules/ccf/Storage.java

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package com.digitalpebble.spruce.modules.ccf;
+
+import com.digitalpebble.spruce.Column;
+import com.digitalpebble.spruce.EnrichmentModule;
+import org.apache.spark.sql.Row;
+
+import java.util.Map;
+
+import static com.digitalpebble.spruce.CURColumn.*;
+import static com.digitalpebble.spruce.CarbonaraColumn.ENERGY_USED;
+
+/**
+ * Provides an estimate of energy used for storage.
+ * Applies a flat coefficient per Gb
+ * @see <a href="https://www.cloudcarbonfootprint.org/docs/methodology#storage">CCF methodology</a>
+ * @see <a href="https://github.com/cloud-carbon-footprint/cloud-carbon-footprint/blob/9f2cf436e5ad020830977e52c3b0a1719d20a8b9/packages/aws/src/lib/CostAndUsageTypes.ts#L25">resource file</a>
+ **/
+public class Storage implements EnrichmentModule {
+
+    //  0.65 Watt-Hours per Terabyte-Hour for HDD
+    final double hdd_gb_coefficient = 0.65 / 1024d;
+    //  1.2 Watt-Hours per Terabyte-Hour for SSD
+    final double ssd_gb_coefficient = 1.2 / 1024d;
+
+    @Override
+    public Column[] columnsAdded() {
+        return new Column[]{ENERGY_USED};
+    }
+
+    @Override
+    public Row process(Row row) {
+        String operation = LINE_ITEM_OPERATION.getString(row);
+        if (operation == null || !operation.startsWith("CreateVolume")) {
+            return row;
+        }
+
+        // in gb months
+        double amount_gb = USAGE_AMOUNT.getDouble(row);
+
+        //  work out which coefficient should be applied
+        // if the line item operation is CreateVolume without a suffix then it is hdd, sdd otherwise
+        // (https://docs.aws.amazon.com/ebs/latest/userguide/ebs-volume-types.html)
+        boolean isHDD = operation.equals("CreateVolume");
+
+        double coefficient = isHDD? hdd_gb_coefficient : ssd_gb_coefficient;
+
+        double energy_gb = amount_gb * coefficient;
+
+        return EnrichmentModule.withUpdatedValue(row, ENERGY_USED, energy_gb);
+    }
+}

src/test/java/com/digitalpebble/spruce/modules/ccf/StorageTest.java

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package com.digitalpebble.spruce.modules.ccf;
+
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema;
+import org.apache.spark.sql.types.StructType;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class StorageTest {
+
+    private final Storage storage = new Storage();
+
+    @Test
+    void processCreateVolumeSSD() {
+        String ddl = "line_item_operation STRING, line_item_usage_amount DOUBLE, energy_usage_kwh DOUBLE";
+        StructType schema = StructType.fromDDL(ddl);
+        Object[] values = new Object[] {"CreateVolume-Gp3", 10d, null};
+        Row row = new GenericRowWithSchema(values, schema);
+        Row result = storage.process(row);
+        double expected = 10d *  storage.ssd_gb_coefficient;
+        assertEquals(expected, result.getDouble(2));
+    }
+
+    @Test
+    void processCreateVolumeSSD2() {
+        String ddl = "line_item_operation STRING, line_item_usage_amount DOUBLE, energy_usage_kwh DOUBLE";
+        StructType schema = StructType.fromDDL(ddl);
+        Object[] values = new Object[] {"CreateVolume-Gp2", 10d, null};
+        Row row = new GenericRowWithSchema(values, schema);
+        Row result = storage.process(row);
+        double expected = 10d *  storage.ssd_gb_coefficient;
+        assertEquals(expected, result.getDouble(2));
+    }
+
+    @Test
+    void processCreateVolumeHDD() {
+        String ddl = "line_item_operation STRING, line_item_usage_amount DOUBLE, energy_usage_kwh DOUBLE";
+        StructType schema = StructType.fromDDL(ddl);
+        Object[] values = new Object[] {"CreateVolume", 10d, null};
+        Row row = new GenericRowWithSchema(values, schema);
+        Row result = storage.process(row);
+        double expected = 10d *  storage.hdd_gb_coefficient;
+        assertEquals(expected, result.getDouble(2));
+    }
+}