Add optional param for AWS IAM role assumption (#143)

Upgrade minio version and add user & policy initialization

Fix permissions

Organize imports

Author: hhamalai

.drone.yml

@@ -17,11 +17,15 @@ steps:
       - git fetch --tags
 
   - name: configure-buckets
-    image: minio/mc:RELEASE.2018-09-26T00-42-43Z
+    image: minio/mc:RELEASE.2020-10-03T02-54-56Z
     commands:
       - sleep 5
       - mc config host add minio http://minio:9000 AKIAIOSFODNN7EXAMPLE wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       - mc mb --region=eu-west-1 minio/drone-cache-bucket
+      - mc admin user add minio foo barbarbar
+      - "echo '{\"Version\": \"2012-10-17\", \"Statement\": [ { \"Action\": [ \"s3:GetObject\", \"s3:PutObject\", \"s3:DeleteObject\", \"s3:CreateBucket\", \"s3:DeleteBucket\" ], \"Effect\": \"Allow\", \"Resource\": [ \"arn:aws:s3:::s3-round-trip-with-role/*\", \"arn:aws:s3:::s3-round-trip-with-role\" ], \"Sid\": \"\" } ] }' >> /tmp/policy.json"
+      - mc admin policy add minio userpolicy /tmp/policy.json
+      - mc admin policy set minio userpolicy user=foo
 
   - name: build
     image: golang:1.14.4-alpine3.12
@@ -206,7 +210,7 @@ steps:
 
 services:
   - name: minio
-    image: minio/minio:RELEASE.2020-03-05T01-04-19Z
+    image: minio/minio:RELEASE.2020-11-06T23-17-07Z
     commands:
       - minio server /data
     environment:

CHANGELOG.md

@@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Fixes [#132](https://github.com/meltwater/drone-cache/issues/132)
 - [#138](https://github.com/meltwater/drone-cache/pull/138) backend/gcs: Fix GCS to pass credentials correctly when `GCS_ENDPOINT` is not set.
 - [#135](https://github.com/meltwater/drone-cache/issues/135) backend/gcs: Fixed parsing of GCS JSON key.
-
+- [#142](https://github.com/meltwater/drone-cache/issues/142) backend/s3: Add option to assume AWS IAM role
 ### Added
 
 - [#102](https://github.com/meltwater/drone-cache/pull/102) Implement option to disable cache rebuild if it already exists in storage.

README.md

@@ -178,6 +178,7 @@ GLOBAL OPTIONS:
    --path-style                          AWS path style to use for bucket paths. (true for minio, false for aws) (default: false) [$PLUGIN_PATH_STYLE, $AWS_PLUGIN_PATH_STYLE]
    --acl value                           upload files with acl (private, public-read, ...) (default: "private") [$PLUGIN_ACL, $AWS_ACL]
    --encryption value                    server-side encryption algorithm, defaults to none. (AES256, aws:kms) [$PLUGIN_ENCRYPTION, $AWS_ENCRYPTION]
+   --role-arn value                      AWS IAM role ARN to assume [$PLUGIN_ASSUME_ROLE_ARN, $AWS_ASSUME_ROLE_ARN]
    --gcs.api-key value                   Google service account API key [$PLUGIN_API_KEY, $GCP_API_KEY]
    --gcs.json-key value                  Google service account JSON key [$PLUGIN_JSON_KEY, $GCS_CACHE_JSON_KEY]
    --gcs.acl value                       upload files with acl (private, public-read, ...) (default: "private") [$PLUGIN_GCS_ACL, $GCS_ACL]

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   minio:
-    image: minio/minio:RELEASE.2020-03-05T01-04-19Z
+    image: minio/minio:RELEASE.2020-11-06T23-17-07Z
     environment:
       MINIO_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE
       MINIO_SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
@@ -25,8 +25,16 @@ services:
     - "10000:10000"
     command: azurite-blob --blobHost 0.0.0.0
   configure-buckets:
-    image: minio/mc:RELEASE.2020-02-20T23-49-54Z
+    image: minio/mc:RELEASE.2020-10-03T02-54-56Z
     entrypoint: sh
     depends_on:
       - minio
-    command: -c "sleep 5 && mc config host add minio http://minio:9000 AKIAIOSFODNN7EXAMPLE wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+    command:
+      - -c
+      - |
+        sleep 2 &&
+        mc config host add minio http://minio:9000 AKIAIOSFODNN7EXAMPLE wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY &&
+        mc admin user add minio foo barbarbar &&
+        echo '{"Version": "2012-10-17", "Statement": [{"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:CreateBucket", "s3:DeleteBucket"], "Effect": "Allow", "Resource": ["arn:aws:s3:::s3-round-trip-with-role/*", "arn:aws:s3:::s3-round-trip-with-role"], "Sid": ""}]}' > /tmp/policy.json &&
+        mc admin policy add minio userpolicy /tmp/policy.json &&
+        mc admin policy set minio userpolicy user=foo

main.go

@@ -363,6 +363,12 @@ func main() {
 			Usage:   "server-side encryption algorithm, defaults to none. (AES256, aws:kms)",
 			EnvVars: []string{"PLUGIN_ENCRYPTION", "AWS_ENCRYPTION"},
 		},
+		&cli.StringFlag{
+			Name:    "role-arn",
+			Usage:   "AWS IAM role ARN to assume",
+			Value:   "",
+			EnvVars: []string{"PLUGIN_ASSUME_ROLE_ARN", "AWS_ASSUME_ROLE_ARN"},
+		},
 
 		// GCS specific Configs flags
 

storage/backend/s3/config.go

@@ -15,6 +15,7 @@ type Config struct {
 	Encryption string // if not "", enables server-side encryption. valid values are: AES256, aws:kms.
 	Endpoint   string
 	Key        string
+	RoleArn    string // if "", do not assume IAM role i.e. use the IAM user.
 
 	// us-east-1
 	// us-west-1

storage/backend/s3/s3.go

@@ -9,9 +9,11 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/aws/aws-sdk-go/service/sts"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 
@@ -44,6 +46,12 @@ func New(l log.Logger, c Config, debug bool) (*Backend, error) {
 		level.Warn(l).Log("msg", "aws key and/or Secret not provided (falling back to anonymous credentials)")
 	}
 
+	if c.RoleArn != "" {
+		conf.Credentials = credentials.NewStaticCredentials(c.Key, c.Secret, "")
+		crds := assumeRole(l, conf, c.RoleArn)
+		conf.Credentials = credentials.NewStaticCredentials(crds.AccessKeyID, crds.SecretAccessKey, crds.SessionToken)
+	}
+
 	level.Debug(l).Log("msg", "s3 backend", "config", fmt.Sprintf("%#v", c))
 
 	if debug {
@@ -138,3 +146,20 @@ func (b *Backend) Exists(ctx context.Context, p string) (bool, error) {
 	// Minio can return success status for without ETag, detect that here.
 	return *out.ETag != "", nil
 }
+
+func assumeRole(l log.Logger, c *aws.Config, roleArn string) credentials.Value {
+	client := sts.New(session.Must(session.NewSessionWithOptions(session.Options{})), c)
+
+	stsProvider := stscreds.AssumeRoleProvider{
+		Client:          client,
+		RoleARN:         roleArn,
+		RoleSessionName: "drone-cache",
+	}
+
+	role, err := stsProvider.Retrieve()
+	if err != nil {
+		level.Error(l).Log("msg", "s3 backend", "assume-role", err.Error())
+	}
+
+	return role
+}

storage/backend/s3/s3_test.go

@@ -20,26 +20,58 @@ import (
 )
 
 const (
-	defaultEndpoint        = "127.0.0.1:9000"
-	defaultAccessKey       = "AKIAIOSFODNN7EXAMPLE"
-	defaultSecretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
-	defaultRegion          = "eu-west-1"
-	defaultACL             = "private"
+	defaultEndpoint            = "127.0.0.1:9000"
+	defaultAccessKey           = "AKIAIOSFODNN7EXAMPLE"
+	defaultSecretAccessKey     = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+	defaultRegion              = "eu-west-1"
+	defaultACL             	   = "private"
+	defaultUserAccessKey   	   = "foo"
+	defaultUserSecretAccessKey = "barbarbar"
 )
 
 var (
-	endpoint        = getEnv("TEST_S3_ENDPOINT", defaultEndpoint)
-	accessKey       = getEnv("TEST_S3_ACCESS_KEY", defaultAccessKey)
-	secretAccessKey = getEnv("TEST_S3_SECRET_KEY", defaultSecretAccessKey)
-	acl             = getEnv("TEST_S3_ACL", defaultACL)
+	endpoint        	= getEnv("TEST_S3_ENDPOINT", defaultEndpoint)
+	accessKey       	= getEnv("TEST_S3_ACCESS_KEY", defaultAccessKey)
+	secretAccessKey 	= getEnv("TEST_S3_SECRET_KEY", defaultSecretAccessKey)
+	acl             	= getEnv("TEST_S3_ACL", defaultACL)
+	userAccessKey       = getEnv("TEST_USER_S3_ACCESS_KEY", defaultUserAccessKey)
+	userSecretAccessKey = getEnv("TEST_USER_S3_SECRET_KEY", defaultUserSecretAccessKey)
 )
 
 func TestRoundTrip(t *testing.T) {
 	t.Parallel()
 
-	backend, cleanUp := setup(t)
+	backend, cleanUp := setup(t, Config{
+		ACL:       acl,
+		Bucket:    "s3-round-trip",
+		Endpoint:  endpoint,
+		Key:       accessKey,
+		PathStyle: true, // Should be true for minio and false for AWS.
+		Region:    defaultRegion,
+		Secret:    secretAccessKey,
+	})
+	t.Cleanup(cleanUp)
+	roundTrip(t, backend)
+}
+
+func TestRoundTripWithAssumeRole(t *testing.T) {
+	t.Parallel()
+
+	backend, cleanUp := setup(t, Config{
+		ACL:       acl,
+		Bucket:    "s3-round-trip-with-role",
+		Endpoint:  endpoint,
+		Key:       userAccessKey,
+		PathStyle: true, // Should be true for minio and false for AWS.
+		Region:    defaultRegion,
+		Secret:    userSecretAccessKey,
+		RoleArn:   "arn:aws:iam::account-id:role/TestRole",
+	})
 	t.Cleanup(cleanUp)
+	roundTrip(t, backend)
+}
 
+func roundTrip(t *testing.T, backend *Backend) {
 	content := "Hello world4"
 
 	// Test Put
@@ -62,44 +94,35 @@ func TestRoundTrip(t *testing.T) {
 
 // Helpers
 
-func setup(t *testing.T) (*Backend, func()) {
-	client := newClient()
-	bucket := "s3-round-trip"
+func setup(t *testing.T, config Config) (*Backend, func()) {
+	client := newClient(config)
 
 	_, err := client.CreateBucketWithContext(context.Background(), &s3.CreateBucketInput{
-		Bucket: aws.String(bucket),
+		Bucket: aws.String(config.Bucket),
 	})
 	test.Ok(t, err)
 
 	b, err := New(
 		log.NewNopLogger(),
-		Config{
-			ACL:       acl,
-			Bucket:    bucket,
-			Endpoint:  endpoint,
-			Key:       accessKey,
-			PathStyle: true, // Should be true for minio and false for AWS.
-			Region:    defaultRegion,
-			Secret:    secretAccessKey,
-		},
+		config,
 		false,
 	)
 	test.Ok(t, err)
 
 	return b, func() {
-		_, _ = client.DeleteBucket(&s3.DeleteBucketInput{
-			Bucket: aws.String(bucket),
+		_, err = client.DeleteBucket(&s3.DeleteBucketInput{
+			Bucket: aws.String(config.Bucket),
 		})
 	}
 }
 
-func newClient() *s3.S3 {
+func newClient(config Config) *s3.S3 {
 	conf := &aws.Config{
 		Region:           aws.String(defaultRegion),
 		Endpoint:         aws.String(endpoint),
 		DisableSSL:       aws.Bool(!strings.HasPrefix(endpoint, "https://")),
 		S3ForcePathStyle: aws.Bool(true),
-		Credentials:      credentials.NewStaticCredentials(accessKey, secretAccessKey, ""),
+		Credentials:      credentials.NewStaticCredentials(config.Key, config.Secret, ""),
 	}
 
 	return s3.New(session.Must(session.NewSessionWithOptions(session.Options{})), conf)