Merge pull request #125 from vinaykns/winc-147

[wni][azure][aws] Open port 10250 via firewall

tools/windows-node-installer/go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/gogo/protobuf v1.2.1 // indirect
 	github.com/googleapis/gnostic v0.3.1 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
+	github.com/masterzen/winrm v0.0.0-20190308153735-1d17eaf15943
 	github.com/openshift/api v3.9.1-0.20190814194116-a94e914914f4+incompatible
 	github.com/openshift/client-go v0.0.0-20190813201236-5a5508328169
 	github.com/spf13/cobra v0.0.5

tools/windows-node-installer/go.sum

@@ -27,7 +27,11 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/Azure/go-ntlmssp v0.0.0-20180810175552-4a21cbd618b4 h1:pSm8mp0T2OH2CPmPDPtwHPr3VAQaOwVF/JbllOPP4xA=
+github.com/Azure/go-ntlmssp v0.0.0-20180810175552-4a21cbd618b4/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022 h1:y8Gs8CzNfDF5AZvjr+5UyGQvQEBL7pwo+v+wX6q9JI8=
+github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.23.2 h1:QSdnxlC29v6b2+C6mkriHhElh02ZlsRBoPX15SOZ6jU=
 github.com/aws/aws-sdk-go v1.23.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
@@ -60,6 +64,8 @@ github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54=
 github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
+github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -93,6 +99,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9 h1:SmVbOZFWAlyQshuMfOkiAx1f5oUTsOGG5IXplAEYeeM=
+github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc=
+github.com/masterzen/winrm v0.0.0-20190308153735-1d17eaf15943 h1:Bteu9XN1gkBePnKr0v1edkUo2LJRsmK5ne2FrC6yVW4=
+github.com/masterzen/winrm v0.0.0-20190308153735-1d17eaf15943/go.mod h1:bsMsaiOA3CXjbJxW0a94G4PfPDj9zUmH5JoFuJ9P4o0=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -146,11 +156,13 @@ go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
 golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=

tools/windows-node-installer/pkg/cloudprovider/aws/ec2.go

@@ -145,13 +145,16 @@ func (a *AwsProvider) CreateWindowsVM() (credentials *types.Credentials, rerr er
 		return nil, fmt.Errorf("failed to get cluster worker IAM, %v", err)
 	}
 
-	// PowerShell script to setup WinRM for Ansible and installing OpenSSH server on the Windows node created
+	// PowerShell script to setup WinRM for Ansible, installing OpenSSH server and open firewall
+	// port number 10250 on the Windows node created
 	userDataWinrm := `<powershell>
         $url = "https://raw.githubusercontent.com/ansible/ansible/devel/examples/scripts/ConfigureRemotingForAnsible.ps1"
         $file = "$env:temp\ConfigureRemotingForAnsible.ps1"
         (New-Object -TypeName System.Net.WebClient).DownloadFile($url,  $file)
-        powershell.exe -ExecutionPolicy ByPass -File $file
-        powershell -NonInteractive -ExecutionPolicy Bypass Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
+        & $file
+        Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
+        New-NetFirewallRule -DisplayName "` + types.FirewallRuleName + `"
+        -Direction Inbound -Action Allow -Protocol TCP -LocalPort ` + types.ContainerLogsPort + ` -EdgeTraversalPolicy Allow
         </powershell>
         <persist>true</persist>`
 

tools/windows-node-installer/pkg/cloudprovider/azure/vm.go

@@ -520,7 +520,7 @@ func (az *AzureProvider) getIPAddress(ctx context.Context) (ipAddress *string, e
 }
 
 // constructAdditionalContent constructs the commands needed to be executed on first login into the Windows node.
-func constructAdditionalContent(instanceName, adminUserName, adminPassword string) *[]compute.AdditionalUnattendContent {
+func constructAdditionalContent(instanceName, adminPassword string) *[]compute.AdditionalUnattendContent {
 	// On first time Logon it will copy the custom file injected to a temporary directory
 	// on windows node, and then it will execute the steps inside the custom script
 	// which will configure winRM Https & Http listeners running on port 5986 & 5985 respectively.
@@ -546,7 +546,7 @@ func constructAdditionalContent(instanceName, adminUserName, adminPassword strin
 			"</FirstLogonCommands>"
 
 	autoLogonData := fmt.Sprintf("<AutoLogon><Domain>%s</Domain><Username>%s</Username><Password><Value>%s</Value></Password>"+
-		"<LogonCount>1</LogonCount><Enabled>true</Enabled></AutoLogon>", instanceName, adminUserName, adminPassword)
+		"<LogonCount>1</LogonCount><Enabled>true</Enabled></AutoLogon>", instanceName, winUser, adminPassword)
 	additionalContent := &[]compute.AdditionalUnattendContent{
 		{
 			// OobeSystem is a configuration setting that is applied during the end-user first boot experience, also
@@ -577,15 +577,17 @@ func constructAdditionalContent(instanceName, adminUserName, adminPassword strin
 // such as configuring remote management listeners, instance access setup.
 func (az *AzureProvider) constructOSProfile(ctx context.Context) (osProfile *compute.OSProfile, vmName, password string) {
 	instanceName := windowsWorker + randomString(5)
-	adminUserName := "core"
 	adminPassword := randomPasswordString(12)
-	additionalContent := constructAdditionalContent(instanceName, adminUserName, adminPassword)
+	additionalContent := constructAdditionalContent(instanceName, adminPassword)
+
 	// the data runs the script from the url location, script sets up both HTTP & HTTPS WinRM listeners so that
-	// Ansible can connect to it and run remote scripts on the windows node.
+	// ansible can connect to it and run remote scripts on the windows node. Also open firewall port number 10250.
 	data := `$url = "https://raw.githubusercontent.com/ansible/ansible/devel/examples/scripts/ConfigureRemotingForAnsible.ps1"
-        $file = "$env:temp\ConfigureRemotingForAnsible.ps1"
-        (New-Object -TypeName System.Net.WebClient).DownloadFile($url,  $file)
-        powershell.exe -ExecutionPolicy ByPass -File $file`
+    $file = "$env:temp\ConfigureRemotingForAnsible.ps1"
+    (New-Object -TypeName System.Net.WebClient).DownloadFile($url,  $file)
+    & $file
+    New-NetFirewallRule -DisplayName "` + types.FirewallRuleName + `"
+    -Direction Inbound -Action Allow -Protocol TCP -LocalPort ` + types.ContainerLogsPort + ` - EdgeTraversalPolicy Allow`
 
 	var nodeLocation string
 	if !checkForNil(az.getvnetLocation(ctx)) {
@@ -594,7 +596,7 @@ func (az *AzureProvider) constructOSProfile(ctx context.Context) (osProfile *com
 	timeZoneMap := getTimeZoneMap()
 	osProfile = &compute.OSProfile{
 		ComputerName:  to.StringPtr(instanceName),
-		AdminUsername: to.StringPtr(adminUserName),
+		AdminUsername: to.StringPtr(winUser),
 		AdminPassword: to.StringPtr(adminPassword),
 		CustomData:    to.StringPtr(base64.StdEncoding.EncodeToString([]byte(data))),
 		WindowsConfiguration: &compute.WindowsConfiguration{

tools/windows-node-installer/pkg/types/types.go

@@ -4,6 +4,13 @@ package types
 // sub-package
 // TODO: Move every cloud provider types here
 
+const (
+	// ContainerLogsPort number will be opened on the windows node via firewall
+	ContainerLogsPort = "10250"
+	// FirewallRuleName is the firewall rule name to open the Container Logs Port
+	FirewallRuleName = "ContainerLogsPort"
+)
+
 // Credentials holds the information to access the Windows instance created.
 type Credentials struct {
 	// instanceID uniquely identifies the instanceID
@@ -16,7 +23,7 @@ type Credentials struct {
 	user string
 }
 
-// NewCredentials takes the instanceID, ip address and password of the Windows instance created and returns the
+// NewCredentials takes the instanceID, ip address, password and user of the Windows instance created and returns the
 // Credentials structure
 func NewCredentials(instanceID, ipAddress, password, user string) *Credentials {
 	return &Credentials{instanceID: instanceID, ipAddress: ipAddress, password: password, user: user}

tools/windows-node-installer/test/e2e/aws_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -18,7 +19,7 @@ var (
 	// Get kubeconfig, AWS credentials, and artifact dir from environment variable set by the OpenShift CI operator.
 	kubeconfig     = os.Getenv("KUBECONFIG")
 	awscredentials = os.Getenv("AWS_SHARED_CREDENTIALS_FILE")
-	dir            = os.Getenv("ARTIFACT_DIR")
+	artifactDir    = os.Getenv("ARTIFACT_DIR")
 	privateKeyPath = os.Getenv("KUBE_SSH_KEY_PATH")
 
 	// The CI-operator uses AWS region `us-east-1` which has the corresponding image ID: ami-0105f663dc99752af for
@@ -52,13 +53,15 @@ func TestAwsE2eSerial(t *testing.T) {
 
 // testCreateWindowsInstance tests the creation of a Windows instance and checks its properties and attached items.
 func testCreateWindowsInstance(t *testing.T) {
+	t.Run("test if instance status is ok", testInstanceStatusOk)
 	t.Run("test created instance properties", testInstanceProperties)
 	t.Run("test instance is attached a public subnet", testInstanceHasPublicSubnetAndIp)
 	t.Run("test instance has name tag", testInstanceIsAttachedWithName)
 	t.Run("test instance has infrastructure tag", testInstanceInfraTagExists)
 	t.Run("test instance is attached the cluster worker's security group", testInstanceHasClusterWorkerSg)
 	t.Run("test instance is attache a Windows security group", testInstanceIsAssociatedWithWindowsWorkerSg)
 	t.Run("test instance is associated with cluster worker's IAM", testInstanceIsAssociatedWithClusterWorkerIAM)
+	t.Run("test container logs port is open in Windows firewall", testInstanceFirewallRule)
 }
 
 // testDestroyWindowsInstance tests the deletion of a Windows instance and checks if the created instance and Windows
@@ -81,7 +84,7 @@ func awsSetup(t *testing.T) {
 // This is the first step of the e2e test and fails the test upon error.
 func setupAWSCloudProvider(t *testing.T) {
 	// The e2e test uses Microsoft Windows Server 2019 Base with Containers image, m4.large type, and libra ssh key.
-	cloud, err := cloudprovider.CloudProviderFactory(kubeconfig, awscredentials, "default", dir,
+	cloud, err := cloudprovider.CloudProviderFactory(kubeconfig, awscredentials, "default", artifactDir,
 		imageID, instanceType, sshKey, privateKeyPath)
 	require.NoError(t, err, "Error obtaining aws interface object")
 
@@ -95,10 +98,12 @@ func setupAWSCloudProvider(t *testing.T) {
 // createdInstanceID, and createdSgID. All information updates are required to be successful or instance will be
 // teared down.
 func setupWindowsInstanceWithResources(t *testing.T) {
-	credentials, err := awsProvider.CreateWindowsVM()
+	var err error
+	credentials, err = awsProvider.CreateWindowsVM()
 	if err != nil {
 		tearDownInstance(t, "error creating Windows instance", err)
 	}
+	require.NoError(t, err, "failed to create the windows instance")
 
 	infraID, err = awsProvider.GetInfraID()
 	if err != nil {
@@ -109,7 +114,7 @@ func setupWindowsInstanceWithResources(t *testing.T) {
 	require.NotEmpty(t, credentials.GetInstanceId(), "Expected instanceID to be present but got empty string")
 
 	// Check instance and security group information in windows-node-installer.json.
-	info, err := resource.ReadInstallerInfo(dir + "/" + "windows-node-installer.json")
+	info, err := resource.ReadInstallerInfo(artifactDir + "/" + "windows-node-installer.json")
 	if err != nil {
 		tearDownInstance(t, "error reading from windows-node-installer.json file", err)
 	}
@@ -127,6 +132,63 @@ func setupWindowsInstanceWithResources(t *testing.T) {
 	}
 }
 
+// waitForStatusok waits for the instance to be okay.
+func waitForStatusOk(instanceId string) error {
+	for i := 0; i < retryCount; i++ {
+		instanceStatus, err := getInstanceStatus(instanceId)
+		if err != nil {
+			fmt.Errorf("failed to get the status of the instance: %v", err)
+		}
+		if instanceStatus == "ok" {
+			return nil
+		}
+		time.Sleep(retryInterval)
+	}
+	return fmt.Errorf("failed to obtain the ok status")
+}
+
+// getInstanceStatus returns the status of the instance.
+func getInstanceStatus(instanceId string) (string, error) {
+	ec2Svc := awsProvider.EC2
+	input := &ec2.DescribeInstanceStatusInput{
+		DryRun:              nil,
+		Filters:             nil,
+		IncludeAllInstances: nil,
+		InstanceIds: []*string{
+			&instanceId,
+		},
+		MaxResults: nil,
+		NextToken:  nil,
+	}
+	result, err := ec2Svc.DescribeInstanceStatus(input)
+	if err != nil {
+		return "", fmt.Errorf("failed to DescribeInstanceStatus with error: %v", err)
+	}
+	if result.InstanceStatuses == nil {
+		return "", fmt.Errorf("InstanceStatuses is nil")
+	}
+
+	// currently we are creating only a single instance which is going to be the
+	// zeroth entry in it.
+	if result.InstanceStatuses[0] == nil {
+		return "", fmt.Errorf("InstanceStatuses[0] is nil")
+	}
+	if result.InstanceStatuses[0].InstanceStatus == nil {
+		return "", fmt.Errorf("windows InstanceStatus is nil")
+	}
+	if result.InstanceStatuses[0].InstanceStatus.Status == nil {
+		return "", fmt.Errorf("windows InstanceStatus.Status field is nil")
+	}
+	instanceStatus := *result.InstanceStatuses[0].InstanceStatus.Status
+	return instanceStatus, nil
+}
+
+// testInstanceStatusOk tests if instance status is ok or not.
+func testInstanceStatusOk(t *testing.T) {
+	err := waitForStatusOk(credentials.GetInstanceId())
+	require.NoErrorf(t, err, "failed to get the instance status as ok")
+}
+
 // getInstance gets the instance information from AWS based on instance ID and returns errors if fails.
 func getInstance(instanceID string) (*ec2.Instance, error) {
 	instances, err := awsProvider.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
@@ -341,6 +403,6 @@ func testSgIsDeleted(t *testing.T) {
 // testInstallerJsonFileIsDeleted asserts that the windows-node-installer.json is deleted.
 func testInstallerJsonFileIsDeleted(t *testing.T) {
 	// the windows-node-installer.json should be removed after resource is deleted.
-	_, err := resource.ReadInstallerInfo(dir)
+	_, err := resource.ReadInstallerInfo(artifactDir)
 	assert.Error(t, err, "error deleting windows-node-installer.json file")
 }