Support for Lambda Images (#2241)

Co-authored-by: Hunter Werlla <werlla@amazon.com>
Co-authored-by: Austin Brooks <abrooksv@users.noreply.github.com>

Author: abrooksv

.changes/next-release/feature-6452c979-bae9-4bb8-bda1-e66f9be413cd.json

@@ -0,0 +1,4 @@
+{
+  "type" : "feature",
+  "description" : "Container Image Support in Lambda"
+}

.gitignore

@@ -10,3 +10,5 @@ Build/
 venv/
 guitest.log
 stale_outputs_checked
+# sam build directory
+.aws-sam/

buildSrc/build.gradle.kts

@@ -59,5 +59,10 @@ gradlePlugin {
             id = "toolkit-change-log"
             implementationClass = "software.aws.toolkits.gradle.changelog.ChangeLogPlugin"
         }
+
+        create("generateSdk") {
+            id = "toolkit-generate-sdk"
+            implementationClass = "software.aws.toolkits.gradle.sdk.GenerateSdkPlugin"
+        }
     }
 }

buildSrc/src/software/aws/toolkits/gradle/sdk/GenerateSdk.kt

@@ -4,54 +4,110 @@
 package software.aws.toolkits.gradle.sdk
 
 import org.gradle.api.DefaultTask
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.plugins.JavaPlugin
+import org.gradle.api.plugins.JavaPluginConvention
 import org.gradle.api.tasks.InputDirectory
+import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.OutputDirectory
+import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.TaskAction
+import org.gradle.plugins.ide.idea.model.IdeaModel
 import software.amazon.awssdk.codegen.C2jModels
 import software.amazon.awssdk.codegen.CodeGenerator
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig
 import software.amazon.awssdk.codegen.model.service.Paginators
 import software.amazon.awssdk.codegen.model.service.ServiceModel
+import software.amazon.awssdk.codegen.model.service.Waiters
 import software.amazon.awssdk.codegen.utils.ModelLoaderUtils
-import java.io.File
+
+open class GenerateSdkExtension(project: Project) {
+    val c2jFolder: DirectoryProperty = project.objects.directoryProperty()
+
+    val outputDir: DirectoryProperty = project.objects.directoryProperty()
+
+    fun srcDir() = outputDir.dir("src")
+    fun tsDir() = outputDir.dir("tst")
+}
+
+@Suppress("unused") // Plugin is created by buildSrc/build.gradle
+class GenerateSdkPlugin : Plugin<Project> {
+    override fun apply(project: Project) {
+        project.pluginManager.apply(JavaPlugin::class.java)
+
+        val extension = project.extensions.create("sdkGenerator", GenerateSdkExtension::class.java, project)
+        extension.c2jFolder.convention(project.layout.projectDirectory.dir("codegen-resources"))
+        extension.outputDir.convention(project.layout.buildDirectory.dir("generated-sources"))
+
+        val generateSdkTask = project.tasks.create("generateSdk", GenerateSdk::class.java)
+
+        val javaConvention = project.convention.getPlugin(JavaPluginConvention::class.java)
+        val mainSourceSet = javaConvention.sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME)
+        mainSourceSet.java.srcDir(generateSdkTask.srcDir)
+        project.tasks.getByName(mainSourceSet.compileJavaTaskName).dependsOn(generateSdkTask)
+
+        val ideaModel = project.extensions.getByType(IdeaModel::class.java)
+        ideaModel.module.generatedSourceDirs.add(generateSdkTask.srcDir.get().asFile)
+    }
+}
 
 open class GenerateSdk : DefaultTask() {
     @InputDirectory
-    lateinit var c2jFolder: File
+    val c2jFolder: DirectoryProperty = project.objects.directoryProperty().convention(project.extensions.getByType(GenerateSdkExtension::class.java).c2jFolder)
 
     @OutputDirectory
-    lateinit var outputDir: File
+    val srcDir: DirectoryProperty = project.objects.directoryProperty().convention(project.extensions.getByType(GenerateSdkExtension::class.java).srcDir())
+
+    @Internal
+    val testDir: DirectoryProperty = project.objects.directoryProperty().convention(project.extensions.getByType(GenerateSdkExtension::class.java).tsDir())
 
     @TaskAction
     fun generate() {
-        outputDir.deleteRecursively()
+        val srcDir = srcDir.asFile.get()
+        val testDir = testDir.asFile.get()
+        srcDir.deleteRecursively()
+        testDir.deleteRecursively()
 
-        logger.info("Generating SDK from $c2jFolder")
+        logger.info("Generating SDK from ${c2jFolder.get().asFile}")
         val models = C2jModels.builder()
             .serviceModel(loadServiceModel())
             .paginatorsModel(loadPaginatorsModel())
             .customizationConfig(loadCustomizationConfig())
+            .waitersModel(loadWaitersModel())
             .build()
 
         CodeGenerator.builder()
             .models(models)
-            .sourcesDirectory(outputDir.absolutePath)
+            .sourcesDirectory(srcDir.absolutePath)
+            .testsDirectory(testDir.absolutePath)
             .build()
             .execute()
     }
 
-    private fun loadServiceModel(): ServiceModel? =
-        ModelLoaderUtils.loadModel(ServiceModel::class.java, File(c2jFolder, "service-2.json"))
+    private fun loadServiceModel(): ServiceModel? = ModelLoaderUtils.loadModel(ServiceModel::class.java, c2jFolder.file("service-2.json").get().asFile)
 
     private fun loadPaginatorsModel(): Paginators? {
-        val paginatorsFile = File(c2jFolder, "paginators-1.json")
-        if (paginatorsFile.exists())
-            return ModelLoaderUtils.loadModel(Paginators::class.java, paginatorsFile)
-        return null
+        val paginatorsFile = c2jFolder.file("paginators-1.json").orNull?.asFile
+        return if (paginatorsFile?.exists() == true) {
+            ModelLoaderUtils.loadModel(Paginators::class.java, paginatorsFile)
+        } else {
+            null
+        }
+    }
+
+    private fun loadWaitersModel(): Waiters? {
+        val waitersFile = c2jFolder.file("waiters-2.json").orNull?.asFile
+        return if (waitersFile?.exists() == true) {
+            ModelLoaderUtils.loadModel(Waiters::class.java, waitersFile)
+        } else {
+            null
+        }
     }
 
     private fun loadCustomizationConfig(): CustomizationConfig = ModelLoaderUtils.loadOptionalModel(
         CustomizationConfig::class.java,
-        File(c2jFolder, "customization.config")
+        c2jFolder.file("customization.config").get().asFile
     ).orElse(CustomizationConfig.create())
 }

core/tst/software/aws/toolkits/core/rules/S3TemporaryBucketRule.kt

@@ -8,22 +8,23 @@ import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.services.s3.model.NoSuchBucketException
 import software.aws.toolkits.core.s3.deleteBucketAndContents
 import software.aws.toolkits.core.utils.RuleUtils
-import software.aws.toolkits.core.utils.Waiters.waitUntilBlocking
 
-class S3TemporaryBucketRule(private val s3Client: S3Client) : ExternalResource() {
+class S3TemporaryBucketRule(private val s3ClientSupplier: () -> S3Client) : ExternalResource() {
+    constructor(s3Client: S3Client) : this({ s3Client })
+
     private val buckets = mutableListOf<String>()
 
     /**
      * Creates a temporary bucket with the optional prefix (or calling class if prefix is omitted)
      */
     fun createBucket(prefix: String = RuleUtils.prefixFromCallingClass()): String {
         val bucketName: String = RuleUtils.randomName(prefix).toLowerCase()
-        s3Client.createBucket { it.bucket(bucketName) }
+        val client = s3ClientSupplier()
+
+        client.createBucket { it.bucket(bucketName) }
 
         // Wait for bucket to be ready
-        waitUntilBlocking(exceptionsToIgnore = setOf(NoSuchBucketException::class)) {
-            s3Client.headBucket { it.bucket(bucketName) }
-        }
+        client.waiter().waitUntilBucketExists { it.bucket(bucketName) }
 
         buckets.add(bucketName)
 
@@ -38,7 +39,7 @@ class S3TemporaryBucketRule(private val s3Client: S3Client) : ExternalResource()
     }
 
     private fun deleteBucketAndContents(bucket: String): Exception? = try {
-        s3Client.deleteBucketAndContents(bucket)
+        s3ClientSupplier().deleteBucketAndContents(bucket)
         null
     } catch (e: Exception) {
         when (e) {

core/tst/software/aws/toolkits/core/utils/IntegrationTestCredentials.kt

@@ -0,0 +1,30 @@
+// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package software.aws.toolkits.core.utils
+
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider
+import software.amazon.awssdk.services.sts.StsClient
+import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest
+import java.util.UUID
+
+/**
+ * Creates an [AwsCredentialsProvider] meant to be used in integration tests
+ *
+ * If the environment variable `ASSUME_ROLE_ARN` is set, it will be assumed using the default credential chain as the source credentials.
+ * If it is not set, we will just use the default credential provider chain
+ */
+fun createIntegrationTestCredentialProvider(): AwsCredentialsProvider {
+    val defaultCredentials = DefaultCredentialsProvider.create()
+
+    return System.getenv("ASSUME_ROLE_ARN")?.takeIf { it.isNotEmpty() }?.let { role ->
+        val sessionName = UUID.randomUUID().toString()
+
+        StsAssumeRoleCredentialsProvider.builder()
+            .stsClient(StsClient.builder().credentialsProvider(defaultCredentials).build())
+            .refreshRequest(AssumeRoleRequest.builder().roleArn(role).roleSessionName(sessionName).build())
+            .build()
+    } ?: defaultCredentials
+}

jetbrains-core/build.gradle.kts

@@ -48,7 +48,7 @@ configurations {
 }
 
 val generateTelemetry = tasks.register<GenerateTelemetry>("generateTelemetry") {
-    inputFiles = listOf()
+    inputFiles = listOf(file("${project.projectDir}/resources/telemetryOverride.json"))
     outputDirectory = file("${project.buildDir}/generated-src")
 }
 compileKotlin.dependsOn(generateTelemetry)
@@ -79,10 +79,12 @@ tasks.jar {
 dependencies {
     api(project(":core"))
     api("software.amazon.awssdk:s3:$awsSdkVersion")
-    api("software.amazon.awssdk:lambda:$awsSdkVersion")
+//    api("software.amazon.awssdk:lambda:$awsSdkVersion") // TODO: Restore back to standard SDK post-launch
+    api(project(":lambda-client"))
     api("software.amazon.awssdk:iam:$awsSdkVersion")
     api("software.amazon.awssdk:ecr:$awsSdkVersion")
     api("software.amazon.awssdk:ecs:$awsSdkVersion")
+    api("software.amazon.awssdk:ecr:$awsSdkVersion")
     api("software.amazon.awssdk:cloudformation:$awsSdkVersion")
     api("software.amazon.awssdk:schemas:$awsSdkVersion")
     api("software.amazon.awssdk:cloudwatchlogs:$awsSdkVersion")

jetbrains-core/it/software/aws/toolkits/jetbrains/services/clouddebug/CloudDebugTestCase.kt

@@ -30,6 +30,7 @@ import software.aws.toolkits.jetbrains.services.ecs.resources.EcsResources
 import software.aws.toolkits.jetbrains.services.ecs.waitForServicesInactive
 import software.aws.toolkits.jetbrains.services.ecs.waitForServicesStable
 import software.aws.toolkits.jetbrains.utils.rules.CloudFormationLazyInitRule
+import java.nio.file.Paths
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.TimeUnit
 
@@ -40,7 +41,7 @@ abstract class CloudDebugTestCase(private val taskDefName: String) {
 
     private val cfnRule = CloudFormationLazyInitRule(
         "CloudDebugTestCluster",
-        CloudDebugTestCase::class.java.getResource("/cloudDebugTestCluster.yaml").readText(),
+        Paths.get(System.getProperty("testDataPath"), "testFiles", "cloudDebugTestCluster.yaml").toFile().readText(),
         emptyList(),
         cloudFormationClient
     )

jetbrains-core/it/software/aws/toolkits/jetbrains/services/lambda/deploy/SamDeployTest.kt

@@ -20,10 +20,12 @@ import software.aws.toolkits.core.region.AwsRegion
 import software.aws.toolkits.core.rules.S3TemporaryBucketRule
 import software.aws.toolkits.jetbrains.core.credentials.MockAwsConnectionManager
 import software.aws.toolkits.jetbrains.core.credentials.runUnderRealCredentials
+import software.aws.toolkits.jetbrains.utils.assumeImageSupport
 import software.aws.toolkits.jetbrains.utils.rules.HeavyJavaCodeInsightTestFixtureRule
 import software.aws.toolkits.jetbrains.utils.setSamExecutableFromEnvironment
 import java.io.File
 import java.nio.file.Paths
+import java.time.Year
 import java.util.UUID
 import java.util.concurrent.TimeUnit
 
@@ -129,6 +131,13 @@ class SamDeployTest {
         }
     }
 
+    @Test
+    fun deployImageBasedSamApp() {
+        assumeImageSupport()
+        // TODO write this test once we have cfn support (or blow up if we take too long :D)
+        assertThat(Year.now().value).isEqualTo(2020)
+    }
+
     private fun setUpProject(templateFilePath: String? = null): VirtualFile {
         projectRule.fixture.addFileToProject(
             "hello_world/app.py",
@@ -181,6 +190,7 @@ class SamDeployTest {
                     templateFile,
                     parameters,
                     bucketRule.createBucket(stackName),
+                    null,
                     false,
                     true,
                     CreateCapabilities.values().toList()