feat(api): add SNMP configuration

As OpenConfig doesn't have SNMP configuration, we use the IETF YANG model defined in
RFC 7407: https://datatracker.ietf.org/doc/rfc7407/

This review includes:
- An ingestor from the NetBox-Network-CMDB SNMP API endpoint.
- Generate IETF Go code from the YANG model.
- A converter to map NetBox-Network-CMDB to the YANG model.
- Create new endpoints:
  - /v1/devices/:hostname/ietfconfig
  - /v1/devices/:hostname/config

Author: cpaillet

Diff for: .golangci.yml

@@ -16,6 +16,7 @@ run:
   skip-files:
     - internal/model/openconfig/oc_path.go
     - internal/model/openconfig/oc.go
+    - internal/model/ietf/ietf.go
 
   # Invariable parameters #
 

Diff for: Makefile

@@ -14,5 +14,5 @@ run:
                  -X 'github.com/criteo/data-aggregation-api/internal/version.buildTime=$(BUILDDATE)'" \
                 ./cmd/data-aggregation-api
 
-update_openconfig:
-	./update_openconfig.sh
+update_yang_models:
+	./update_yang_models.sh

Diff for: README.md

@@ -35,4 +35,4 @@ You need to update the following part in the code:
 4. add your convertor in `internal/convertors/...`
 
 3. execute your convertor (`internal/convertors/device/device.go`):
-   - GenerateOpenconfig()
+   - Generateconfigs()

Diff for: internal/api/router/endpoints.go

@@ -92,6 +92,56 @@ func (m *Manager) getDeviceOpenConfig(w http.ResponseWriter, _ *http.Request, ps
 	_, _ = w.Write(cfg)
 }
 
+// getDeviceIETFConfig endpoint returns Ietf JSON for one or all devices.
+func (m *Manager) getDeviceIETFConfig(w http.ResponseWriter, _ *http.Request, ps httprouter.Params) {
+	w.Header().Set(contentType, applicationJSON)
+	hostname := ps.ByName(hostnameKey)
+	if ps.ByName(hostnameKey) == wildcard {
+		cfg, err := m.devices.GetAllDevicesIETFConfigJSON()
+		if err != nil {
+			log.Error().Err(err).Send()
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		_, _ = w.Write(cfg)
+		return
+	}
+
+	cfg, err := m.devices.GetDeviceIETFConfigJSON(hostname)
+	if err != nil {
+		log.Error().Err(err).Send()
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	_, _ = w.Write(cfg)
+}
+
+// getDeviceConfig endpoint returns Ietf & openconfig JSON for one or all devices.
+func (m *Manager) getDeviceConfig(w http.ResponseWriter, _ *http.Request, ps httprouter.Params) {
+	w.Header().Set(contentType, applicationJSON)
+	hostname := ps.ByName(hostnameKey)
+	if ps.ByName(hostnameKey) == wildcard {
+		cfg, err := m.devices.GetAllDevicesConfigJSON()
+		if err != nil {
+			log.Error().Err(err).Send()
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		_, _ = w.Write(cfg)
+		return
+	}
+
+	cfg, err := m.devices.GetDeviceConfigJSON(hostname)
+	if err != nil {
+		log.Error().Err(err).Send()
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	_, _ = w.Write(cfg)
+}
+
 // getLastReport returns the last or current report.
 func (m *Manager) getLastReport(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
 	out, err := m.reports.GetLastJSON()

Diff for: internal/api/router/manager.go

@@ -24,6 +24,10 @@ type DevicesRepository interface {
 	IsAFKEnabledJSON(hostname string) ([]byte, error)
 	GetAllDevicesOpenConfigJSON() ([]byte, error)
 	GetDeviceOpenConfigJSON(hostname string) ([]byte, error)
+	GetAllDevicesIETFConfigJSON() ([]byte, error)
+	GetDeviceIETFConfigJSON(hostname string) ([]byte, error)
+	GetAllDevicesConfigJSON() ([]byte, error)
+	GetDeviceConfigJSON(hostname string) ([]byte, error)
 }
 
 type Manager struct {
@@ -56,6 +60,8 @@ func (m *Manager) ListenAndServe(ctx context.Context, address string, port int)
 	router.GET("/api/health", healthCheck)
 	router.GET("/v1/devices/:hostname/afk_enabled", withAuth.Wrap(m.getAFKEnabled))
 	router.GET("/v1/devices/:hostname/openconfig", withAuth.Wrap(m.getDeviceOpenConfig))
+	router.GET("/v1/devices/:hostname/ietfconfig", withAuth.Wrap(m.getDeviceIETFConfig))
+	router.GET("/v1/devices/:hostname/config", withAuth.Wrap(m.getDeviceConfig))
 	router.GET("/v1/report/last", withAuth.Wrap(m.getLastReport))
 	router.GET("/v1/report/last/complete", withAuth.Wrap(m.getLastCompleteReport))
 	router.GET("/v1/report/last/successful", withAuth.Wrap(m.getLastSuccessfulReport))

Diff for: internal/convertor/device/device.go

@@ -8,10 +8,13 @@ import (
 
 	bgpconvertors "github.com/criteo/data-aggregation-api/internal/convertor/bgp"
 	rpconvertors "github.com/criteo/data-aggregation-api/internal/convertor/routingpolicy"
+	snmpconvertors "github.com/criteo/data-aggregation-api/internal/convertor/snmp"
 	"github.com/criteo/data-aggregation-api/internal/ingestor/repository"
 	"github.com/criteo/data-aggregation-api/internal/model/cmdb/bgp"
 	"github.com/criteo/data-aggregation-api/internal/model/cmdb/routingpolicy"
+	"github.com/criteo/data-aggregation-api/internal/model/cmdb/snmp"
 	"github.com/criteo/data-aggregation-api/internal/model/dcim"
+	"github.com/criteo/data-aggregation-api/internal/model/ietf"
 	"github.com/criteo/data-aggregation-api/internal/model/openconfig"
 	"github.com/openconfig/ygot/ygot"
 	"github.com/rs/zerolog/log"
@@ -22,15 +25,18 @@ const AFKEnabledTag = "afk-enabled"
 var defaultInstance = "default"
 
 type GeneratedConfig struct {
-	Openconfig *openconfig.Device
-	JSON       string
+	IETF           *ietf.Device
+	JSONIETF       string
+	Openconfig     *openconfig.Device
+	JSONOpenConfig string
 }
 
 type Device struct {
 	mutex           *sync.Mutex
 	Dcim            *dcim.NetworkDevice
 	Config          *GeneratedConfig
 	BGPGlobalConfig *bgp.BGPGlobal
+	SNMP            *snmp.SNMP
 	Sessions        []*bgp.Session
 	PeerGroups      []*bgp.PeerGroup
 	PrefixLists     []*routingpolicy.PrefixList
@@ -92,12 +98,16 @@ func NewDevice(dcimInfo *dcim.NetworkDevice, devicesData *repository.AssetsPerDe
 		return nil, fmt.Errorf("no route-policies found for %s", dcimInfo.Hostname)
 	}
 
+	device.SNMP, ok = devicesData.SNMP[dcimInfo.Hostname]
+	if !ok {
+		log.Warn().Msgf("no snmp found for %s", dcimInfo.Hostname)
+	}
 	return device, nil
 }
 
-// GenerateOpenconfig generate the OpenConfig data for the current device.
+// Generateconfigs generate the Config (openconfig & ietf) data for the current device.
 // The CMDB data must have been precomputed before running this method.
-func (d *Device) GenerateOpenconfig() error {
+func (d *Device) Generateconfigs() error {
 	d.mutex.Lock()
 	defer d.mutex.Unlock()
 
@@ -139,23 +149,60 @@ func (d *Device) GenerateOpenconfig() error {
 			Indent:         "  ",
 		},
 	)
+
 	if err != nil {
 		return fmt.Errorf("failed to transform an openconfig device specification (%s) into JSON using ygot: %w", d.Dcim.Hostname, err)
 	}
 
 	d.Config = &GeneratedConfig{
-		Openconfig: &config,
-		JSON:       devJSON,
+		Openconfig:     &config,
+		JSONOpenConfig: devJSON,
+		IETF:           nil,
+		JSONIETF:       "{}",
 	}
 
+	if d.SNMP == nil {
+		log.Warn().Msgf("%s don't have a Snmp configuration, skip IetfConfig", d.Dcim.Hostname)
+	} else {
+		IetfSystem := snmpconvertors.SNMPtoIETFfSystem(d.SNMP)
+		IetfSnmp := snmpconvertors.SNMPtoIETFsnmp(d.SNMP)
+
+		d.Config.IETF = &ietf.Device{
+			System: &IetfSystem,
+			Snmp:   &IetfSnmp,
+		}
+
+		d.Config.JSONIETF, err = ygot.EmitJSON(
+			d.Config.IETF,
+			&ygot.EmitJSONConfig{
+				Format:         ygot.RFC7951,
+				SkipValidation: false,
+				Indent:         "  ",
+			},
+		)
+		if err != nil {
+			return fmt.Errorf("failed to transform an ietf device specification (%s) into JSON using ygot: %w", d.Dcim.Hostname, err)
+		}
+	}
 	return nil
 }
 
-// GetCompactJSON returns OpenConfig result in not indented JSON format.
+// GetCompactOpenconfigJSON returns OpenConfig result in not indented JSON format.
 // Generated JSON is already indented by Ygot - currently there is no option to not indent the JSON.
-func (d *Device) GetCompactJSON() ([]byte, error) {
+func (d *Device) GetCompactOpenconfigJSON() ([]byte, error) {
+	out := bytes.NewBuffer(nil)
+	err := json.Compact(out, []byte(d.Config.JSONOpenConfig))
+	if err != nil {
+		return nil, err
+	}
+	return out.Bytes(), nil
+}
+
+// GetCompactIETFJSON returns IETF result in not indented JSON format.
+// GetCompactIETFJSON JSON is already indented by Ygot - currently there is no option to not indent the JSON.
+func (d *Device) GetCompactIETFJSON() ([]byte, error) {
 	out := bytes.NewBuffer(nil)
-	err := json.Compact(out, []byte(d.Config.JSON))
+	err := json.Compact(out, []byte(d.Config.JSONIETF))
 	if err != nil {
 		return nil, err
 	}