bugfix: provider version coming as 0.0.0 or empty (#1553)

Author: nasir-rabbani

pkg/iac-providers/terraform/commons/load-dir.go

@@ -158,9 +158,6 @@ func (t TerraformDirectoryLoader) loadDirRecursive(dirList []string) (output.All
 			t.addError(errMessage, dir)
 		}
 
-		// getting provider version for the root module
-		providerVersion := GetModuleProviderVersion(rootMod)
-
 		// get unified config for the current directory
 		unified, diags := t.buildUnifiedConfig(rootMod, dir)
 		// Get the downloader chache
@@ -209,12 +206,7 @@ func (t TerraformDirectoryLoader) loadDirRecursive(dirList []string) (output.All
 				}
 
 				resourceConfig.TerraformVersion = t.terraformVersion
-				resourceConfig.ProviderVersion = providerVersion
-
-				// if root module do not have provider contraints fetch the latest compatible version
-				if resourceConfig.ProviderVersion == "" {
-					resourceConfig.ProviderVersion = LatestProviderVersion(managedResource.Provider, t.terraformVersion)
-				}
+				resourceConfig.ProviderVersion = GetModuleProviderVersion(current.Config.Module, managedResource.Provider, t.terraformVersion)
 				// set module name
 				resourceConfig.ModuleName = current.Name
 
@@ -295,9 +287,6 @@ func (t TerraformDirectoryLoader) loadDirNonRecursive() (output.AllResourceConfi
 		t.addError(errMessage, t.absRootDir)
 	}
 
-	// getting provider version for the root module
-	providerVersion := GetModuleProviderVersion(rootMod)
-
 	// get unified config for the current directory
 	unified, diags := t.buildUnifiedConfig(rootMod, t.absRootDir)
 
@@ -360,12 +349,7 @@ func (t TerraformDirectoryLoader) loadDirNonRecursive() (output.AllResourceConfi
 			}
 
 			resourceConfig.TerraformVersion = t.terraformVersion
-			resourceConfig.ProviderVersion = providerVersion
-
-			// if root module do not have provider contraints fetch the latest compatible version
-			if resourceConfig.ProviderVersion == "" {
-				resourceConfig.ProviderVersion = LatestProviderVersion(managedResource.Provider, t.terraformVersion)
-			}
+			resourceConfig.ProviderVersion = GetModuleProviderVersion(current.Config.Module, managedResource.Provider, t.terraformVersion)
 
 			if isRemoteModule {
 				resourceConfig.IsRemoteModule = &isRemoteModule

pkg/iac-providers/terraform/commons/load-file.go

@@ -42,13 +42,11 @@ func LoadIacFile(absFilePath, terraformVersion string) (allResourcesConfig outpu
 		return allResourcesConfig, fmt.Errorf(errMessage)
 	}
 
-	if diags != nil {
+	if diags.HasErrors() {
 		errMessage := fmt.Sprintf("failed to load iac file '%s'. error:\n%v\n", absFilePath, getErrorMessagesFromDiagnostics(diags))
 		zap.S().Debug(errMessage)
 		return allResourcesConfig, fmt.Errorf(errMessage)
 	}
-	// getting provider version for the file
-	providerVersion := GetFileProviderVersion(hclFile)
 
 	// initialize normalized output
 	allResourcesConfig = make(map[string][]output.ResourceConfig)
@@ -62,21 +60,16 @@ func LoadIacFile(absFilePath, terraformVersion string) (allResourcesConfig outpu
 			return allResourcesConfig, fmt.Errorf("failed to create ResourceConfig")
 		}
 
+		resourceConfig.TerraformVersion = terraformVersion
+		managedResource.Provider = ResolveProvider(managedResource, hclFile.RequiredProviders)
+		resourceConfig.ProviderVersion = GetProviderVersion(hclFile, managedResource.Provider, terraformVersion)
 		// set module name
 		// module name for the file scan will always be root
 		resourceConfig.ModuleName = "root"
 
 		// extract file name from path
 		resourceConfig.Source = getFileName(resourceConfig.Source)
 
-		resourceConfig.TerraformVersion = terraformVersion
-		resourceConfig.ProviderVersion = providerVersion
-
-		// if root module do not have provider contraints fetch the latest compatible version
-		if resourceConfig.ProviderVersion == "" {
-			resourceConfig.ProviderVersion = LatestProviderVersion(managedResource.Provider, terraformVersion)
-		}
-
 		// append to normalized output
 		if _, present := allResourcesConfig[resourceConfig.Type]; !present {
 			allResourcesConfig[resourceConfig.Type] = []output.ResourceConfig{resourceConfig}

pkg/iac-providers/terraform/commons/load-file_test.go

@@ -0,0 +1,102 @@
+/*
+    Copyright (C) 2022 Tenable, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package commons
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/tenable/terrascan/pkg/iac-providers/output"
+	"github.com/tenable/terrascan/pkg/iac-providers/terraform/commons/test"
+	"github.com/tenable/terrascan/pkg/utils"
+)
+
+func TestLoadIacFile(t *testing.T) {
+	type args struct {
+		absFilePath      string
+		terraformVersion string
+	}
+	tests := []struct {
+		name                   string
+		args                   args
+		outputJSON             string
+		wantAllResourcesConfig output.AllResourceConfigs
+		wantErr                bool
+	}{
+		{
+			name: "file with no provider defined",
+			args: args{
+				absFilePath:      filepath.Join(testDataDir, "terraform_iac_files", "with_no_provider.tf"),
+				terraformVersion: "0.15.0",
+			},
+			outputJSON: filepath.Join(testDataDir, "tfjson", "output_no_provider_defined.json"),
+		},
+		{
+			name: "file with provider config",
+			args: args{
+				absFilePath:      filepath.Join(testDataDir, "terraform_iac_files", "with_provider_config.tf"),
+				terraformVersion: "0.15.0",
+			},
+			outputJSON: filepath.Join(testDataDir, "tfjson", "output_with_provider_config.json"),
+		},
+		{
+			name: "file with required provider",
+			args: args{
+				absFilePath:      filepath.Join(testDataDir, "terraform_iac_files", "with_required_provider.tf"),
+				terraformVersion: "0.15.0",
+			},
+			outputJSON: filepath.Join(testDataDir, "tfjson", "output_with_required_provider.json"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := LoadIacFile(tt.args.absFilePath, tt.args.terraformVersion)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("LoadIacFile() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			// if !reflect.DeepEqual(gotAllResourcesConfig, tt.wantAllResourcesConfig) {
+			// 	t.Errorf("LoadIacFile() = %v, want %v", gotAllResourcesConfig, tt.wantAllResourcesConfig)
+			// }
+
+			var want output.AllResourceConfigs
+
+			// Read the expected value and unmarshal into want
+			contents, _ := os.ReadFile(tt.outputJSON)
+			if utils.IsWindowsPlatform() {
+				contents = utils.ReplaceWinNewLineBytes(contents)
+			}
+
+			err = json.Unmarshal(contents, &want)
+			if err != nil {
+				t.Errorf("unexpected error unmarshalling want: %v", err)
+			}
+
+			match, err := test.IdenticalAllResourceConfigs(got, want)
+			if err != nil {
+				t.Errorf("unexpected error checking result: %v", err)
+			}
+			if !match {
+				g, _ := json.MarshalIndent(got, "", "  ")
+				w, _ := json.MarshalIndent(want, "", "  ")
+				t.Errorf("got '%v', want: '%v'", string(g), string(w))
+			}
+		})
+	}
+}

pkg/iac-providers/terraform/commons/terraform-provider.go

@@ -10,9 +10,8 @@ import (
 	"strings"
 
 	"github.com/apparentlymart/go-versions/versions"
-	"github.com/apparentlymart/go-versions/versions/constraints"
 	"github.com/hashicorp/terraform/addrs"
-	"github.com/hashicorp/terraform/configs"
+	hclConfigs "github.com/hashicorp/terraform/configs"
 	httputils "github.com/tenable/terrascan/pkg/utils/http"
 	"go.uber.org/zap"
 )
@@ -22,11 +21,7 @@ const (
 	terraformVersionHeader = "X-Terraform-Version"
 )
 
-// VersionConstraints ...
-type VersionConstraints = constraints.IntersectionSpec
-
-// Requirements ...
-type Requirements map[addrs.Provider]VersionConstraints
+var versionCache = make(map[string]string)
 
 // ProviderVersions ...
 type ProviderVersions struct {
@@ -37,14 +32,10 @@ type ProviderVersions struct {
 	Warnings []string `json:"warnings"`
 }
 
-// ProviderVersionList fetches all the versions of terraform providers
-func ProviderVersionList(ctx context.Context, addr addrs.Provider, terraformVersion string) (versions.List, []string, error) {
+// providerVersionList fetches all the versions of terraform providers
+func providerVersionList(ctx context.Context, addr addrs.Provider, terraformVersion string) (versions.List, []string, error) {
 	zap.S().Debugf("fetching list of providers metadata, hostname: %q, type: %q, namespace: %q, ", addr.Hostname.String(), addr.Namespace, addr.Type)
 
-	if addr.Hostname.String() == "" {
-		return nil, nil, fmt.Errorf("error preparing the providers list endpoint, error: hostname can't be empty")
-	}
-
 	endpointURL, err := url.Parse(path.Join(apiVersion, "providers", addr.Namespace, addr.Type, "versions"))
 	if err != nil {
 		return nil, nil, fmt.Errorf("error preparing the providers list endpoint, error: %s", err.Error())
@@ -92,76 +83,108 @@ func ProviderVersionList(ctx context.Context, addr addrs.Provider, terraformVers
 		}
 		versionList = append(versionList, ver)
 	}
-	// versionList.Newest()
-
 	return versionList, body.Warnings, nil
 }
 
-var versionCache = make(map[string]string)
-
-// LatestProviderVersion returns the latest published version for the asked provider.
+// latestProviderVersion returns the latest published version for the asked provider.
 // It returns "0.0.0" in case its not available
-func LatestProviderVersion(addr addrs.Provider, terraformVersion string) string {
+func latestProviderVersion(addr addrs.Provider, terraformVersion string) string {
 
 	// check if the cache has the version info
 	if v, found := versionCache[fmt.Sprintf("%s-%s", addr.Type, terraformVersion)]; found {
 		return v
 	}
-	versionList, _, err := ProviderVersionList(context.TODO(), addr, terraformVersion)
+	versionList, _, err := providerVersionList(context.TODO(), addr, terraformVersion)
 	if err != nil {
-		zap.S().Errorf("failed to fetch latest version for terraform provider, error: %s", err.Error())
+		zap.S().Warnf("failed to fetch latest version for terraform provider, error: %s", err.Error())
+		return versionList.Newest().String()
 	}
 	// update cache
 	versionCache[fmt.Sprintf("%s-%s", addr.Type, terraformVersion)] = versionList.Newest().String()
 	return versionList.Newest().String()
 }
 
-// ParseVersionConstraints parses a "Ruby-like" version constraint string
-// into a VersionConstraints value.
-func ParseVersionConstraints(str string) (VersionConstraints, error) {
-	return constraints.ParseRubyStyleMulti(str)
+// GetProviderVersion identifies the version constraints for file resources.
+func GetProviderVersion(f *hclConfigs.File, addr addrs.Provider, terraformVersion string) string {
+	version := ""
+
+	for _, rps := range f.RequiredProviders {
+		if rp, exist := rps.RequiredProviders[addr.Type]; exist {
+			version = trimVersionConstraints(rp.Requirement.Required.String())
+			break
+		}
+	}
+
+	// older version of terraform (terraform version < 1.x) may have version in provider block
+	if len(version) == 0 {
+		for _, pc := range f.ProviderConfigs {
+			if pc.Name == addr.Type {
+				version = trimVersionConstraints(pc.Version.Required.String())
+				break
+			}
+		}
+	}
+
+	// fetch latest version
+	if len(version) == 0 {
+		version = latestProviderVersion(addr, terraformVersion)
+	}
+	v, err := versions.ParseVersion(version)
+	if err != nil {
+		zap.S().Warnf("failed to parse provider version: %s", err.Error())
+		return ""
+	}
+	return v.String()
 }
 
-// GetModuleProviderVersion gets the provider version form the 'required_providers' block for module.
-// if the 'required_providers' is not defined, it returns empty string
-func GetModuleProviderVersion(m *configs.Module) string {
+// GetModuleProviderVersion identifies the version constraints for module resources.
+func GetModuleProviderVersion(module *hclConfigs.Module, addr addrs.Provider, terraformVersion string) string {
 	version := ""
-	if m == nil || m.ProviderRequirements == nil {
-		return version
+
+	if rp, exist := module.ProviderRequirements.RequiredProviders[addr.Type]; exist {
+		version = trimVersionConstraints(rp.Requirement.Required.String())
 	}
-	for _, requiredProvider := range m.ProviderRequirements.RequiredProviders {
-		if requiredProvider != nil && len(requiredProvider.Requirement.Required) > 0 {
-			version = requiredProvider.Requirement.Required[0].String()
+
+	// older version of terraform (terraform version < 1.x) may have version in provider block
+	if len(version) == 0 {
+		if pc, exist := module.ProviderConfigs[addr.Type]; exist {
+			version = trimVersionConstraints(pc.Version.Required.String())
 		}
 	}
 
-	// trim version string
-	s := strings.Split(version, " ")
-	if len(s) > 1 {
-		version = s[1]
+	// fetch latest version
+	if len(version) == 0 {
+		version = latestProviderVersion(addr, terraformVersion)
 	}
-
-	return version
+	v, err := versions.ParseVersion(version)
+	if err != nil {
+		zap.S().Warnf("failed to parse provider version: %s", err.Error())
+		return ""
+	}
+	return v.String()
 }
 
-// GetFileProviderVersion gets the provider version form the 'required_providers' block for the file.
-// if the 'required_providers' is not defined, it returns empty string
-func GetFileProviderVersion(f *configs.File) string {
-	version := ""
-	if f == nil || f.RequiredProviders == nil || len(f.RequiredProviders) == 0 {
-		return version
+// ResolveProvider resolves provider addr
+func ResolveProvider(resource *hclConfigs.Resource, requiredProviders []*hclConfigs.RequiredProviders) addrs.Provider {
+	implied, err := addrs.ParseProviderPart(resource.Addr().ImpliedProvider())
+	if err != nil {
+		zap.S().Warnf("failed to parse provider namespace or type: %s", err.Error())
+		return addrs.NewDefaultProvider("aws")
 	}
-	for _, requiredProvider := range f.RequiredProviders[0].RequiredProviders {
-		if requiredProvider != nil && len(requiredProvider.Requirement.Required) > 0 {
-			version = requiredProvider.Requirement.Required[0].String()
+	for _, rp := range requiredProviders {
+		if provider, exists := rp.RequiredProviders[implied]; exists {
+			return provider.Type
 		}
 	}
+	return addrs.ImpliedProviderForUnqualifiedType(implied)
+}
 
-	// trim version string
-	s := strings.Split(version, " ")
+// trimVersionConstraints trim version constraints from string.
+// e.g. "~> 3.0.2" will become "3.0.2"
+func trimVersionConstraints(v string) string {
+	s := strings.Split(v, " ")
 	if len(s) > 1 {
-		version = s[1]
+		v = s[1]
 	}
-
-	return version
+	return v
 }