Merge pull request #19 from sethsec/fix_download_from_aws_error

Fix: Handle new AWS API format with hashed service IDs

Author: sethsec

awsservicemap.go

@@ -13,7 +13,14 @@ import (
 var awsJson embed.FS
 
 type serviceEntry struct {
-	ID string `json:"id"`
+	ID         string              `json:"id"`
+	Attributes *serviceAttributes  `json:"attributes,omitempty"`
+}
+
+type serviceAttributes struct {
+	Region      string `json:"aws:region"`
+	ServiceName string `json:"aws:serviceName"`
+	ServiceURL  string `json:"aws:serviceUrl"`
 }
 
 type regionalServiceData struct {
@@ -30,6 +37,28 @@ func contains(element string, array []string) bool {
 	return false
 }
 
+// extractServiceSlug extracts the service identifier from AWS service URL
+// Example: "https://aws.amazon.com/ec2/" -> "ec2"
+//          "https://aws.amazon.com/rds/mysql/" -> "rds"
+//          "https://aws.amazon.com/systems-manager/" -> "systems-manager"
+func extractServiceSlug(serviceURL string) string {
+	if serviceURL == "" {
+		return ""
+	}
+
+	// Remove trailing slash
+	serviceURL = strings.TrimSuffix(serviceURL, "/")
+
+	// Split by / and get the last non-empty part that's not the domain
+	parts := strings.Split(serviceURL, "/")
+	for i := len(parts) - 1; i >= 0; i-- {
+		if parts[i] != "" && !strings.Contains(parts[i], "aws.amazon.com") && !strings.Contains(parts[i], "http") {
+			return parts[i]
+		}
+	}
+	return ""
+}
+
 type AwsServiceMap struct {
 	JsonFileSource JsonFileSource
 	cachedData     *regionalServiceData // Cache the parsed data to avoid repeated HTTP requests
@@ -91,14 +120,18 @@ func (m *AwsServiceMap) parseJson() (regionalServiceData, error) {
 		if err != nil {
 			return serviceData, err
 		}
-		json.Unmarshal([]byte(body), &serviceData)
+
+		err = json.Unmarshal([]byte(body), &serviceData)
 		if err != nil {
 			return serviceData, err
 		}
 
 	} else {
 		jsonFile, err := awsJson.ReadFile("data/aws-service-regions.json")
-		json.Unmarshal([]byte(jsonFile), &serviceData)
+		if err != nil {
+			return serviceData, err
+		}
+		err = json.Unmarshal([]byte(jsonFile), &serviceData)
 		if err != nil {
 			return serviceData, err
 		}
@@ -130,80 +163,148 @@ func (m *AwsServiceMap) GetAllRegions() ([]string, error) {
 
 // Returns a slice of strings that represent all regions that support the specific service
 func (m *AwsServiceMap) GetRegionsForService(reqService string) ([]string, error) {
-	regionsForServiceMap := map[string][]string{}
+	regionsForService := []string{}
 
 	serviceData, err := m.parseJson()
 	if err != nil {
-		return regionsForServiceMap[reqService], err
+		return regionsForService, err
 	}
-	for _, id := range serviceData.ServiceEntries {
-		service := strings.Split(id.ID, ":")[0]
-		if _, ok := regionsForServiceMap[service]; !ok {
-			regionsForServiceMap[service] = nil
+
+	for _, entry := range serviceData.ServiceEntries {
+		idParts := strings.Split(entry.ID, ":")
+		if len(idParts) != 2 {
+			continue
 		}
-		region := strings.Split(id.ID, ":")[1]
-		if _, ok := regionsForServiceMap[service]; ok {
-			regionsForServiceMap[service] = append(regionsForServiceMap[service], region)
+
+		serviceHash := idParts[0]
+		region := idParts[1]
+
+		// Check if this is the service we're looking for
+		var matches bool
+
+		// Try old format first (direct match)
+		if serviceHash == reqService {
+			matches = true
+		} else if entry.Attributes != nil && entry.Attributes.ServiceURL != "" {
+			// New format: extract slug from URL
+			serviceSlug := extractServiceSlug(entry.Attributes.ServiceURL)
+			if serviceSlug == reqService {
+				matches = true
+			}
+		}
+
+		if matches && !contains(region, regionsForService) {
+			regionsForService = append(regionsForService, region)
 		}
 	}
-	return regionsForServiceMap[reqService], err
 
+	return regionsForService, nil
 }
 
 // Returns a slice of strings that represent all observed services
-
 func (m *AwsServiceMap) GetAllServices() ([]string, error) {
 	totalServices := []string{}
 	serviceData, err := m.parseJson()
 	if err != nil {
 		return totalServices, err
 	}
-	for _, id := range serviceData.ServiceEntries {
-		service := strings.Split(id.ID, ":")[0]
-		if !contains(service, totalServices) {
-			totalServices = append(totalServices, service)
+
+	for _, entry := range serviceData.ServiceEntries {
+		var serviceName string
+
+		if entry.Attributes != nil && entry.Attributes.ServiceURL != "" {
+			// New format: extract from URL
+			serviceName = extractServiceSlug(entry.Attributes.ServiceURL)
+		} else {
+			// Fallback to hash from ID (old format)
+			idParts := strings.Split(entry.ID, ":")
+			if len(idParts) > 0 {
+				serviceName = idParts[0]
+			}
+		}
+
+		if serviceName != "" && !contains(serviceName, totalServices) {
+			totalServices = append(totalServices, serviceName)
 		}
 	}
-	return totalServices, err
 
+	return totalServices, nil
 }
 
 // Returns a slice of strings that represent all service supported in a specific region
 func (m *AwsServiceMap) GetServicesForRegion(reqRegion string) ([]string, error) {
-	servicesForRegionMap := map[string][]string{}
+	servicesForRegion := []string{}
 	serviceData, err := m.parseJson()
 	if err != nil {
-		return servicesForRegionMap[reqRegion], err
+		return servicesForRegion, err
 	}
 
-	for _, id := range serviceData.ServiceEntries {
+	for _, entry := range serviceData.ServiceEntries {
+		idParts := strings.Split(entry.ID, ":")
+		if len(idParts) != 2 {
+			continue
+		}
 
-		region := strings.Split(id.ID, ":")[1]
-		if _, ok := servicesForRegionMap[region]; !ok {
-			servicesForRegionMap[region] = nil
+		serviceHash := idParts[0]
+		region := idParts[1]
+
+		if region != reqRegion {
+			continue
 		}
 
-		service := strings.Split(id.ID, ":")[0]
-		if _, ok := servicesForRegionMap[region]; ok {
-			servicesForRegionMap[region] = append(servicesForRegionMap[region], service)
+		// Determine service name
+		var serviceName string
+		if entry.Attributes != nil && entry.Attributes.ServiceURL != "" {
+			serviceName = extractServiceSlug(entry.Attributes.ServiceURL)
+		} else {
+			// Fallback to hash (old format)
+			serviceName = serviceHash
+		}
+
+		if serviceName != "" && !contains(serviceName, servicesForRegion) {
+			servicesForRegion = append(servicesForRegion, serviceName)
 		}
 	}
-	return servicesForRegionMap[reqRegion], err
+
+	return servicesForRegion, nil
 }
 
 // Is a specific service supported in a specific region. Returns true/false
+// Handles both old format (ec2:us-east-1) and new format (hash:us-east-1 with attributes)
 func (m *AwsServiceMap) IsServiceInRegion(reqService string, reqRegion string) (bool, error) {
 	serviceData, err := m.parseJson()
 	if err != nil {
 		return false, err
 	}
 
+	// Try direct match first (old format or if hash is provided)
 	reqPair := serviceEntry{ID: fmt.Sprintf("%s:%s", reqService, reqRegion)}
 	if serviceEntryContains(reqPair, serviceData.ServiceEntries) {
-		return true, err
-	} else {
-		return false, err
+		return true, nil
+	}
+
+	// New format: check if any entry with matching region has the service slug
+	for _, entry := range serviceData.ServiceEntries {
+		// Check if region matches
+		idParts := strings.Split(entry.ID, ":")
+		if len(idParts) != 2 {
+			continue
+		}
+
+		if idParts[1] != reqRegion {
+			continue
+		}
+
+		// Extract service slug from URL if attributes exist
+		if entry.Attributes != nil && entry.Attributes.ServiceURL != "" {
+			serviceSlug := extractServiceSlug(entry.Attributes.ServiceURL)
+			if serviceSlug == reqService {
+				return true, nil
+			}
+		}
 	}
+
+	return false, nil
 }
 
 func serviceEntryContains(element serviceEntry, array []serviceEntry) bool {

cmd/examples/main.go

@@ -16,7 +16,8 @@ func main() {
 	// With the new caching feature, the data is fetched only once per instance regardless of how many method calls are made.
 
 	servicemap := &awsservicemap.AwsServiceMap{
-		JsonFileSource: "EMBEDDED_IN_PACKAGE",
+		//JsonFileSource: "EMBEDDED_IN_PACKAGE",
+		JsonFileSource: "DOWNLOAD_FROM_AWS",
 	}
 
 	//  Example of how you can also use the constructor pattern to simulate "instantiating" a new service map "object"

test/test_new_format.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/bishopfox/awsservicemap"
+)
+
+func main() {
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+
+	servicemap := &awsservicemap.AwsServiceMap{
+		JsonFileSource: "DOWNLOAD_FROM_AWS",
+	}
+
+	fmt.Println("=== Testing awsservicemap with new AWS API format ===\n")
+
+	// Test IsServiceInRegion
+	fmt.Println("1. Testing IsServiceInRegion for EC2:")
+	testRegions := []string{"us-east-1", "us-west-2", "eu-west-1", "ap-southeast-1"}
+	for _, region := range testRegions {
+		res, err := servicemap.IsServiceInRegion("ec2", region)
+		if err != nil {
+			fmt.Printf("   ERROR checking %s: %v\n", region, err)
+		} else {
+			fmt.Printf("   EC2 in %s: %v\n", region, res)
+		}
+	}
+
+	// Test GetRegionsForService
+	fmt.Println("\n2. Testing GetRegionsForService for EC2:")
+	regions, err := servicemap.GetRegionsForService("ec2")
+	if err != nil {
+		fmt.Printf("   ERROR: %v\n", err)
+	} else {
+		fmt.Printf("   Found EC2 in %d regions\n", len(regions))
+		fmt.Printf("   First 5 regions: %v\n", regions[:min(5, len(regions))])
+	}
+
+	// Test GetAllServices
+	fmt.Println("\n3. Testing GetAllServices:")
+	services, err := servicemap.GetAllServices()
+	if err != nil {
+		fmt.Printf("   ERROR: %v\n", err)
+	} else {
+		fmt.Printf("   Found %d total services\n", len(services))
+		fmt.Printf("   First 10 services: %v\n", services[:min(10, len(services))])
+	}
+
+	// Test GetServicesForRegion
+	fmt.Println("\n4. Testing GetServicesForRegion for us-east-1:")
+	usEast1Services, err := servicemap.GetServicesForRegion("us-east-1")
+	if err != nil {
+		fmt.Printf("   ERROR: %v\n", err)
+	} else {
+		fmt.Printf("   Found %d services in us-east-1\n", len(usEast1Services))
+		fmt.Printf("   First 10 services: %v\n", usEast1Services[:min(10, len(usEast1Services))])
+	}
+
+	// Test GetAllRegions
+	fmt.Println("\n5. Testing GetAllRegions:")
+	allRegions, err := servicemap.GetAllRegions()
+	if err != nil {
+		fmt.Printf("   ERROR: %v\n", err)
+	} else {
+		fmt.Printf("   Found %d total regions\n", len(allRegions))
+		fmt.Printf("   All regions: %v\n", allRegions)
+	}
+
+	// Test a few more services
+	fmt.Println("\n6. Testing other services:")
+	otherServices := []string{"rds", "lambda", "s3", "iam", "eks"}
+	for _, svc := range otherServices {
+		res, err := servicemap.IsServiceInRegion(svc, "us-east-1")
+		if err != nil {
+			fmt.Printf("   ERROR checking %s: %v\n", svc, err)
+		} else {
+			fmt.Printf("   %s in us-east-1: %v\n", svc, res)
+		}
+	}
+
+	fmt.Println("\n=== All tests completed successfully! ===")
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}