Adding dmidecode support in server-info for better view of memory subsystem (#178)

* Added(server-info): reminder about installation pcituils and dmidecode.
* Added(server-info): using dmidecode memory data file (type, speed and size). for -show and -rate
* Added(test-data): server-info archive from cheapest vscale CentOS 7
* Added(assessor): Grade by known values
* Fixed(test-data): server-info show expected output changed
* Refactoring(dmi): parser complexity fighting
* Refactoring(parsers): provide optional simplier way of initialisation for any Parser-based class
* Incremented version

#37

Author: strizhechenko

netutils_linux_hardware/assessor.py

@@ -39,12 +39,33 @@ def assess_cpu(self):
                 'Vendor ID': Grade.str(cpuinfo.get('Vendor ID'), good=['GenuineIntel']),
             }
 
+    def assess_memory_device(self, device):
+        return {
+            'size': Grade.int(device.get('size', 0), 512, 8196),
+            'type': Grade.known_values(device.get('type', 'RAM'), {
+                'DDR1': 2,
+                'DDR2': 3,
+                'DDR3': 6,
+                'DDR4': 10,
+            }),
+            'speed': Grade.int(device.get('speed', 0), 200, 4000),
+        }
+
+    def assess_memory_devices(self, devices):
+        return dict((handle, self.assess_memory_device(device)) for handle, device in devices.items())
+
+    def assess_memory_size(self, size):
+        return {
+            'MemTotal': Grade.int(size.get('MemTotal'), 2 * (1024 ** 2), 16 * (1024 ** 2)),
+            'SwapTotal': Grade.int(size.get('SwapTotal'), 512 * 1024, 4 * (1024 ** 2)),
+        }
+
     def assess_memory(self):
         meminfo = self.data.get('memory')
         if meminfo:
             return {
-                'MemTotal': Grade.int(meminfo.get('MemTotal'), 2 * (1024 ** 2), 16 * (1024 ** 2)),
-                'SwapTotal': Grade.int(meminfo.get('SwapTotal'), 512 * 1024, 4 * (1024 ** 2)),
+                'devices': self.assess_memory_devices(meminfo.get('devices')),
+                'size': self.assess_memory_size(meminfo.get('size')),
             }
 
     def assess_system(self):

netutils_linux_hardware/grade.py

@@ -18,3 +18,7 @@ def str(value, good=None, bad=None):
     @staticmethod
     def fact(value, mode=False):
         return 10 if (value is None) != mode else 1
+
+    @staticmethod
+    def known_values(value, _dict):
+        return _dict.get(value, 1)

netutils_linux_hardware/parsers.py

@@ -8,7 +8,9 @@
 
 
 class Parser(object):
-    @staticmethod
+    def __init__(self, filepath=None):
+        self.result = self.parse_file_safe(filepath) if filepath else None
+
     def parse(text, **kwargs):
         raise NotImplementedError
 
@@ -122,6 +124,48 @@ def parse(self, text):
         return dict((k, int(v.replace(' kB', ''))) for k, v in iteritems(yaml.load(text)) if k in self.keys_required)
 
 
+class MemInfoDMIDevice(object):
+    def __init__(self, text):
+        self.data = {
+            'speed': 0,
+            'type': 'RAM',
+            'size': 0,
+        }
+        self.type = 'RAM'
+        self.handle = None
+        self.size = 0
+        self.parse_text(text)
+
+    def parse_text(self, text):
+        """ Разбор описания плашки памяти от dmidecode """
+        for line in map(str.strip, text.split('\n')):
+            self.parse_line(line)
+
+    def parse_line(self, line):
+        for key in ('Speed', 'Type', 'Size'):
+            if line.startswith(key + ':'):
+                self.data[key.lower()] = line.split()[1]
+                break
+        if line.startswith('Handle'):
+            self.handle = line.split(' ')[1].strip(',')
+
+
+class MemInfoDMI(Parser):
+    @staticmethod
+    def parse(text):
+        """ Разбор всего вывода dmidecode --type memory """
+        return MemInfoDMI.__parse(text.split('\n\n')) if text else None
+
+    @staticmethod
+    def __parse(devices):
+        output = dict()
+        for device in devices:
+            if 'Memory Device' in device:
+                mem_dev = MemInfoDMIDevice(device)
+                output[mem_dev.handle] = mem_dev
+        return output
+
+
 class CPULayout(Parser):
     @staticmethod
     def parse(text):

netutils_linux_hardware/reader.py

@@ -2,13 +2,14 @@
 # pylint: disable=C0111, C0103
 
 import os
+
 import yaml
-from netutils_linux_hardware.parsers import YAMLLike, CPULayout, DiskInfo, MemInfo
+
 from netutils_linux_hardware.netdev import ReaderNet
+from netutils_linux_hardware.parsers import YAMLLike, CPULayout, DiskInfo, MemInfo, MemInfoDMI
 
 
 class Reader(object):
-
     info = None
 
     def __init__(self, datadir):
@@ -25,12 +26,15 @@ def gather_info(self):
 
         self.info = {
             'cpu': {
-                'info': YAMLLike().parse_file_safe(self.path('lscpu_info')),
-                'layout': CPULayout().parse_file_safe(self.path('lscpu_layout')),
+                'info': YAMLLike(self.path('lscpu_info')).result,
+                'layout': CPULayout(self.path('lscpu_layout')).result,
             },
             'net': ReaderNet(self.datadir, self.path).netdevs,
             'disk': DiskInfo().parse(self.path('disks_types'), self.path('lsblk_sizes'), self.path('lsblk_models')),
-            'memory': MemInfo().parse_file_safe(self.path('meminfo')),
+            'memory': {
+                'size': MemInfo(self.path('meminfo')).result,
+                'devices': MemInfoDMI(self.path('dmidecode')).result,
+            },
         }
         for key in ('CPU MHz', 'BogoMIPS'):
             if self.info.get('cpu', {}).get('info', {}).get(key):

setup.py

@@ -16,7 +16,7 @@ def read(*paths):
 
 setuptools.setup(
     name='netutils-linux',
-    version='2.5.2',
+    version='2.6.1',
     author='Oleg Strizhechenko',
     author_email='[email protected]',
     license='MIT',

tests/server-info-show.tests/1xE3-1271.I217_and_X710.L2_mixed.masterconf/expected_output

@@ -33,10 +33,12 @@ disk:
     size: 120034123776
     type: SSD
 memory:
-  MemFree: 14142256
-  MemTotal: 16290904
-  SwapFree: 8216572
-  SwapTotal: 8216572
+  devices: null
+  size:
+    MemFree: 14142256
+    MemTotal: 16290904
+    SwapFree: 8216572
+    SwapTotal: 8216572
 net:
   eth0:
     buffers:

tests/server-info-show.tests/1xE31240.2x82576.L2.manual/expected_output

@@ -31,7 +31,9 @@ cpu:
     '6': '0'
     '7': '0'
 disk: null
-memory: null
+memory:
+  devices: null
+  size: null
 net:
   eth0:
     buffers:

tests/server-info-show.tests/1xE5200.82571EB_and_AR8121.L2.manual/expected_output

@@ -24,7 +24,9 @@ cpu:
     '0': '0'
     '1': '0'
 disk: null
-memory: null
+memory:
+  devices: null
+  size: null
 net:
   eth1:
     buffers:

tests/server-info-show.tests/1xi3.single_i350.l2_mixed.manual/expected_output

@@ -28,7 +28,9 @@ cpu:
     '2': '0'
     '3': '0'
 disk: null
-memory: null
+memory:
+  devices: null
+  size: null
 net:
   eth6:
     buffers:

tests/server-info-show.tests/1xi7-4770K.1x82541PI.L2.manual/expected_output

@@ -32,7 +32,9 @@ cpu:
     '6': '0'
     '7': '0'
 disk: null
-memory: null
+memory:
+  devices: null
+  size: null
 net:
   eth0:
     buffers:

tests/server-info-show.tests/2xE5-2640.i350_and_82599ES.l2_mixed.masterconf/expected_output

@@ -65,10 +65,12 @@ disk:
     size: 1000204886016
     type: HDD
 memory:
-  MemFree: 432940
-  MemTotal: 32843852
-  SwapFree: 16490492
-  SwapTotal: 16490492
+  devices: null
+  size:
+    MemFree: 432940
+    MemTotal: 32843852
+    SwapFree: 16490492
+    SwapTotal: 16490492
 net:
   eth1:
     buffers:

tests/server-info-show.tests/2xE5530.82576_and_82574L.l2_mixed.manual/expected_output

@@ -39,7 +39,9 @@ cpu:
     '8': '0'
     '9': '1'
 disk: null
-memory: null
+memory:
+  devices: null
+  size: null
 net:
   eth1:
     buffers:

tests/server-info-show.tests/2xL5520.1x82576.L2_mixed.manual/expected_output

@@ -40,7 +40,9 @@ cpu:
     '8': '0'
     '9': '0'
 disk: null
-memory: null
+memory:
+  devices: null
+  size: null
 net:
   em2:
     buffers:

tests/server-info-show.tests/2xX5650.BCM5709_and_MT27520.L2.manual/expected_output

@@ -36,7 +36,9 @@ cpu:
     '8': '0'
     '9': '1'
 disk: null
-memory: null
+memory:
+  devices: null
+  size: null
 net:
   eth1:
     buffers:

tests/server-info-show.tests/kvm.l3.masterconf/expected_output

@@ -26,7 +26,9 @@ cpu:
     '0': '0'
     '1': '1'
 disk: null
-memory: null
+memory:
+  devices: null
+  size: null
 net:
   eth1:
     buffers:

tests/server-info-show.tests/vscale-vm/cpuinfo

@@ -0,0 +1,26 @@
+processor	: 0
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 79
+model name	: Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz
+stepping	: 1
+microcode	: 0x1
+cpu MHz		: 2199.986
+cache size	: 25600 KB
+physical id	: 0
+siblings	: 1
+core id		: 0
+cpu cores	: 1
+apicid		: 0
+initial apicid	: 0
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 13
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl eagerfpu pni pclmulqdq vmx ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap xsaveopt arat
+bogomips	: 4399.97
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 40 bits physical, 48 bits virtual
+power management:
+

tests/server-info-show.tests/vscale-vm/disks_types

@@ -0,0 +1 @@
+/sys/block/vda/queue/rotational:1

tests/server-info-show.tests/vscale-vm/dmidecode

@@ -0,0 +1,27 @@
+# dmidecode 3.0
+Getting SMBIOS data from sysfs.
+SMBIOS 2.4 present.
+
+Handle 0x1000, DMI type 16, 15 bytes
+Physical Memory Array
+	Location: Other
+	Use: System Memory
+	Error Correction Type: Multi-bit ECC
+	Maximum Capacity: 512 MB
+	Error Information Handle: Not Provided
+	Number Of Devices: 1
+
+Handle 0x1100, DMI type 17, 21 bytes
+Memory Device
+	Array Handle: 0x1000
+	Error Information Handle: 0x0000
+	Total Width: 64 bits
+	Data Width: 64 bits
+	Size: 512 MB
+	Form Factor: DIMM
+	Set: None
+	Locator: DIMM 0
+	Bank Locator: Not Specified
+	Type: RAM
+	Type Detail: None
+

tests/server-info-show.tests/vscale-vm/ethtool/g/eth0

@@ -0,0 +1,12 @@
+Ring parameters for eth0:
+Pre-set maximums:
+RX:		256
+RX Mini:	0
+RX Jumbo:	0
+TX:		256
+Current hardware settings:
+RX:		256
+RX Mini:	0
+RX Jumbo:	0
+TX:		256
+

tests/server-info-show.tests/vscale-vm/ethtool/i/eth0

@@ -0,0 +1,10 @@
+driver: virtio_net
+version: 1.0.0
+firmware-version: 
+expansion-rom-version: 
+bus-info: 0000:00:03.0
+supports-statistics: no
+supports-test: no
+supports-eeprom-access: no
+supports-register-dump: no
+supports-priv-flags: no

tests/server-info-show.tests/vscale-vm/interrupts

@@ -0,0 +1,34 @@
+           CPU0       
+  0:         24   IO-APIC-edge      timer
+  1:         10   IO-APIC-edge      i8042
+  6:          3   IO-APIC-edge      floppy
+  8:          1   IO-APIC-edge      rtc0
+  9:          0   IO-APIC-fasteoi   acpi
+ 10:          0   IO-APIC-fasteoi   virtio2
+ 11:          0   IO-APIC-fasteoi   uhci_hcd:usb1
+ 12:         15   IO-APIC-edge      i8042
+ 14:          0   IO-APIC-edge      ata_piix
+ 15:          0   IO-APIC-edge      ata_piix
+ 24:          0   PCI-MSI-edge      virtio1-config
+ 25:     607727   PCI-MSI-edge      virtio1-req.0
+ 26:          0   PCI-MSI-edge      virtio0-config
+ 27:    6070473   PCI-MSI-edge      virtio0-input.0
+ 28:         17   PCI-MSI-edge      virtio0-output.0
+NMI:          0   Non-maskable interrupts
+LOC:   45350286   Local timer interrupts
+SPU:          0   Spurious interrupts
+PMI:          0   Performance monitoring interrupts
+IWI:    4143286   IRQ work interrupts
+RTR:          0   APIC ICR read retries
+RES:          0   Rescheduling interrupts
+CAL:          0   Function call interrupts
+TLB:          0   TLB shootdowns
+TRM:          0   Thermal event interrupts
+THR:          0   Threshold APIC interrupts
+DFR:          0   Deferred Error APIC interrupts
+MCE:          0   Machine check exceptions
+MCP:       3760   Machine check polls
+ERR:          0
+MIS:          0
+PIN:          0   Posted-interrupt notification event
+PIW:          0   Posted-interrupt wakeup event

tests/server-info-show.tests/vscale-vm/lsblk_models

@@ -0,0 +1,2 @@
+NAME MODEL
+vda  

tests/server-info-show.tests/vscale-vm/lsblk_sizes

@@ -0,0 +1,2 @@
+21474836480            vda
+21473771008 /          vda1