Add mon groups for resctrl.

"mon_groups" can be created to monitor subsets of tasks in the CTRL_MON
group that is their ancestor.

More info: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt

Signed-off-by: Paweł Szulik <pawel.szulik@intel.com>

Author: Paweł Szulik

events.go

@@ -154,12 +154,12 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 	}
 
 	if is := ls.IntelRdtStats; is != nil {
-		if intelrdt.IsCatEnabled() {
+		if intelrdt.IsCatEnabled() && is.L3CacheInfo != nil {
 			s.IntelRdt.L3CacheInfo = convertL3CacheInfo(is.L3CacheInfo)
 			s.IntelRdt.L3CacheSchemaRoot = is.L3CacheSchemaRoot
 			s.IntelRdt.L3CacheSchema = is.L3CacheSchema
 		}
-		if intelrdt.IsMbaEnabled() {
+		if intelrdt.IsMbaEnabled() && is.MemBwInfo != nil{
 			s.IntelRdt.MemBwInfo = convertMemBwInfo(is.MemBwInfo)
 			s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot
 			s.IntelRdt.MemBwSchema = is.MemBwSchema

libcontainer/SPEC.md

@@ -158,22 +158,31 @@ init process will block waiting for the parent to finish setup.
 ### IntelRdt
 
 Intel platforms with new Xeon CPU support Resource Director Technology (RDT).
-Cache Allocation Technology (CAT) and Memory Bandwidth Allocation (MBA) are
-two sub-features of RDT.
+Cache Allocation Technology (CAT), Cache Monitoring Technology (CMT),
+Memory Bandwidth Allocation (MBA) and Memory Bandwidth Monitoring (MBM) are
+four sub-features of RDT.
 
 Cache Allocation Technology (CAT) provides a way for the software to restrict
 cache allocation to a defined 'subset' of L3 cache which may be overlapping
 with other 'subsets'. The different subsets are identified by class of
 service (CLOS) and each CLOS has a capacity bitmask (CBM).
 
+Cache Monitoring Technology (CMT) supports monitoring of the last-level cache (LLC)
+for each running thread simultaneously.
+
 Memory Bandwidth Allocation (MBA) provides indirect and approximate throttle
 over memory bandwidth for the software. A user controls the resource by
-indicating the percentage of maximum memory bandwidth or memory bandwidth limit
-in MBps unit if MBA Software Controller is enabled.
+indicating the percentage of maximum memory bandwidth or memory bandwidth
+limit in MBps unit if MBA Software Controller is enabled.
+
+Memory Bandwidth Monitoring (MBM) supports monitoring of memory bandwidth total and local
+for each running thread simultaneously.
 
-It can be used to handle L3 cache and memory bandwidth resources allocation
-for containers if hardware and kernel support Intel RDT CAT and MBA features.
+More details about Intel RDT CAT and MBA can be found in the section 17.18
+of Intel Software Developer Manual:
+https://software.intel.com/en-us/articles/intel-sdm
 
+About Intel RDT kernel interface:
 In Linux 4.10 kernel or newer, the interface is defined and exposed via
 "resource control" filesystem, which is a "cgroup-like" interface.
 
@@ -185,7 +194,6 @@ CAT and MBA features are introduced in Linux 4.10 and 4.12 kernel via
 "resource control" filesystem.
 
 Intel RDT "resource control" filesystem hierarchy:
-```
 mount -t resctrl resctrl /sys/fs/resctrl
 tree /sys/fs/resctrl
 /sys/fs/resctrl/
@@ -194,22 +202,40 @@ tree /sys/fs/resctrl
 |   |   |-- cbm_mask
 |   |   |-- min_cbm_bits
 |   |   |-- num_closids
+|   |-- L3_MON
+|   |   |-- max_threshold_occupancy
+|   |   |-- mon_features
+|   |   |-- num_rmids
 |   |-- MB
 |       |-- bandwidth_gran
 |       |-- delay_linear
 |       |-- min_bandwidth
 |       |-- num_closids
-|-- ...
+|-- mon_data
+    |-- mon_L3_00
+    	   |-- llc_occupancy
+    	   |-- mbm_local_bytes
+        |-- mbm_total_bytes
+    |-- ...
+|-- mon_groups
+    |-- <container_id>
+        |-- ...
+        |-- mon_data
+    	       |-- mon_L3_00
+    	           |-- llc_occupancy
+                |-- mbm_local_bytes
+                |-- mbm_total_bytes
+            |-- ...
 |-- schemata
 |-- tasks
 |-- <container_id>
     |-- ...
     |-- schemata
     |-- tasks
-```
 
 For runc, we can make use of `tasks` and `schemata` configuration for L3
-cache and memory bandwidth resources constraints.
+cache and memory bandwidth resources constraints, `mon_data` directory for
+CMT and MBM statistics.
 
 The file `tasks` has a list of tasks that belongs to this group (e.g.,
 <container_id>" group). Tasks can be added to a group by writing the task ID
@@ -223,9 +249,7 @@ Each resource (L3 cache, memory bandwidth) has its own line and format.
 L3 cache schema:
 It has allocation bitmasks/values for L3 cache on each socket, which
 contains L3 cache id and capacity bitmask (CBM).
-```
 	Format: "L3:<cache_id0>=<cbm0>;<cache_id1>=<cbm1>;..."
-```
 For example, on a two-socket machine, the schema line could be "L3:0=ff;1=c0"
 which means L3 cache id 0's CBM is 0xff, and L3 cache id 1's CBM is 0xc0.
 
@@ -239,9 +263,7 @@ set in a group: 0xf, 0xf0, 0x3ff, 0x1f00 and etc.
 Memory bandwidth schema:
 It has allocation values for memory bandwidth on each socket, which contains
 L3 cache id and memory bandwidth.
-```
 	Format: "MB:<cache_id0>=bandwidth0;<cache_id1>=bandwidth1;..."
-```
 For example, on a two-socket machine, the schema line could be "MB:0=20;1=70"
 
 The minimum bandwidth percentage value for each CPU model is predefined and
@@ -251,7 +273,7 @@ that is allocated is also dependent on the CPU model and can be looked up at
 min_bw + N * bw_gran. Intermediate values are rounded to the next control
 step available on the hardware.
 
-If MBA Software Controller is enabled through mount option "-o mba_MBps"
+If MBA Software Controller is enabled through mount option "-o mba_MBps":
 mount -t resctrl resctrl -o mba_MBps /sys/fs/resctrl
 We could specify memory bandwidth in "MBps" (Mega Bytes per second) unit
 instead of "percentages". The kernel underneath would use a software feedback
@@ -263,11 +285,12 @@ For example, on a two-socket machine, the schema line could be
 "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on socket 0
 and 7000 MBps memory bandwidth limit on socket 1.
 
-For more information about Intel RDT kernel interface:  
+For more information about Intel RDT kernel interface:
 https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt
 
-```
+
 An example for runc:
+```
 Consider a two-socket machine with two L3 caches where the default CBM is
 0x7ff and the max CBM length is 11 bits, and minimum memory bandwidth of 10%
 with a memory bandwidth granularity of 10%.
@@ -278,10 +301,26 @@ maximum memory bandwidth of 20% on socket 0 and 70% on socket 1.
 
 "linux": {
     "intelRdt": {
-        "closID": "guaranteed_group",
         "l3CacheSchema": "L3:0=7f0;1=1f",
         "memBwSchema": "MB:0=20;1=70"
-    }
+	}
+}
+```
+Another example:
+```
+We only want to monitor memory bandwidth and llc occupancy.
+"linux": {
+    "intelRdt": {
+	}
+}
+
+or
+
+"linux": {
+    "intelRdt": {
+        "l3CacheSchema": "",
+        "memBwSchema": ""
+	}
 }
 ```
 

libcontainer/configs/config.go

@@ -192,7 +192,7 @@ type Config struct {
 	NoNewKeyring bool `json:"no_new_keyring"`
 
 	// IntelRdt specifies settings for Intel RDT group that the container is placed into
-	// to limit the resources (e.g., L3 cache, memory bandwidth) the container has available
+	// to limit the resources (e.g., L3 cache, memory bandwidth) the container has available.
 	IntelRdt *IntelRdt `json:"intel_rdt,omitempty"`
 
 	// RootlessEUID is set when the runc was launched with non-zero EUID.

libcontainer/configs/validate/validator.go

@@ -182,23 +182,31 @@ func (v *ConfigValidator) sysctl(config *configs.Config) error {
 
 func (v *ConfigValidator) intelrdt(config *configs.Config) error {
 	if config.IntelRdt != nil {
+		/*
 		if !intelrdt.IsCatEnabled() && !intelrdt.IsMbaEnabled() {
 			return errors.New("intelRdt is specified in config, but Intel RDT is not supported or enabled")
 		}
-
+		*/
 		if !intelrdt.IsCatEnabled() && config.IntelRdt.L3CacheSchema != "" {
 			return errors.New("intelRdt.l3CacheSchema is specified in config, but Intel RDT/CAT is not enabled")
 		}
 		if !intelrdt.IsMbaEnabled() && config.IntelRdt.MemBwSchema != "" {
 			return errors.New("intelRdt.memBwSchema is specified in config, but Intel RDT/MBA is not enabled")
 		}
 
+		if !intelrdt.IsMBMEnabled() && config.IntelRdt.L3CacheSchema == "" && config.IntelRdt.MemBwSchema == "" {
+			return errors.New("intelRdt is pecified in config, but Intel RDT/MBM is not enabled")
+		}
+
+		/*
 		if intelrdt.IsCatEnabled() && config.IntelRdt.L3CacheSchema == "" {
 			return errors.New("Intel RDT/CAT is enabled and intelRdt is specified in config, but intelRdt.l3CacheSchema is empty")
 		}
 		if intelrdt.IsMbaEnabled() && config.IntelRdt.MemBwSchema == "" {
 			return errors.New("Intel RDT/MBA is enabled and intelRdt is specified in config, but intelRdt.memBwSchema is empty")
 		}
+
+		 */
 	}
 
 	return nil

libcontainer/container_linux.go

@@ -1852,10 +1852,6 @@ func (c *linuxContainer) currentState() (*State, error) {
 		startTime, _ = c.initProcess.startTime()
 		externalDescriptors = c.initProcess.externalDescriptors()
 	}
-	intelRdtPath, err := intelrdt.GetIntelRdtPath(c.ID())
-	if err != nil {
-		intelRdtPath = ""
-	}
 	state := &State{
 		BaseState: BaseState{
 			ID:                   c.ID(),
@@ -1866,7 +1862,7 @@ func (c *linuxContainer) currentState() (*State, error) {
 		},
 		Rootless:            c.config.RootlessEUID && c.config.RootlessCgroups,
 		CgroupPaths:         c.cgroupManager.GetPaths(),
-		IntelRdtPath:        intelRdtPath,
+		IntelRdtPath:        c.intelRdtManager.GetPath(),
 		NamespacePaths:      make(map[configs.NamespaceType]string),
 		ExternalDescriptors: externalDescriptors,
 	}

libcontainer/factory_linux.go

@@ -313,14 +313,13 @@ func (l *LinuxFactory) Load(id string) (Container, error) {
 		cgroupManager:        l.NewCgroupsManager(state.Config.Cgroups, state.CgroupPaths),
 		root:                 containerRoot,
 		created:              state.Created,
+		intelRdtManager:      l.NewIntelRdtManager(&state.Config, id, state.IntelRdtPath),
 	}
 	c.state = &loadedState{c: c}
 	if err := c.refreshState(); err != nil {
 		return nil, err
 	}
-	if intelrdt.IsCatEnabled() || intelrdt.IsMbaEnabled() {
-		c.intelRdtManager = l.NewIntelRdtManager(&state.Config, id, state.IntelRdtPath)
-	}
+
 	return c, nil
 }