packer: pick protobuf/gob for serialisation (#13025)

As we're trying to move away from gob for serialising data over the
wire, this commit adds the capability for Packer to pick dynamically
between gob or protobuf for the serialisation format to communicate with
plugins.

As it stands, if all the plugins discovered are compatible with
protobuf, and we have not forced gob usage, protobuf will be the
serialisation format picked.

If any plugin is not compatible with protobuf, gob will be used for
communicating with all the plugins that will be used over the course of
a command.

command/build.go

@@ -105,6 +105,11 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
 	diags = packerStarter.Initialize(packer.InitializeOptions{
 		UseSequential: cla.UseSequential,
 	})
+
+	if packer.PackerUseProto {
+		log.Printf("[TRACE] Using protobuf for communication with plugins")
+	}
+
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
 		return ret

command/execute.go

@@ -5,8 +5,8 @@
 package command
 
 import (
+	"flag"
 	"fmt"
-	"log"
 	"regexp"
 	"strings"
 
@@ -75,18 +75,45 @@ var Datasources = map[string]packersdk.Datasource{
 
 var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner|datasource)-(.+)")
 
-func (c *ExecuteCommand) Run(args []string) int {
-	// This is an internal call (users should not call this directly) so we're
-	// not going to do much input validation. If there's a problem we'll often
-	// just crash. Error handling should be added to facilitate debugging.
-	log.Printf("args: %#v", args)
+type ExecuteArgs struct {
+	UseProtobuf bool
+	CommandType string
+}
+
+func (ea *ExecuteArgs) AddFlagSets(flags *flag.FlagSet) {
+	flags.BoolVar(&ea.UseProtobuf, "protobuf", false, "Use protobuf for serialising data over the wire instead of gob")
+}
+
+func (c *ExecuteCommand) ParseArgs(args []string) (*ExecuteArgs, int) {
+	var cfg ExecuteArgs
+	flags := c.Meta.FlagSet("")
+	flags.Usage = func() { c.Ui.Say(c.Help()) }
+	cfg.AddFlagSets(flags)
+	if err := flags.Parse(args); err != nil {
+		return &cfg, 1
+	}
+
+	args = flags.Args()
 	if len(args) != 1 {
-		c.Ui.Error(c.Help())
-		return 1
+		flags.Usage()
+		return &cfg, 1
 	}
+	cfg.CommandType = args[0]
+	return &cfg, 0
+}
 
+func (c *ExecuteCommand) Run(args []string) int {
+	cfg, ret := c.ParseArgs(args)
+	if ret != 0 {
+		return ret
+	}
+
+	return c.RunContext(cfg)
+}
+
+func (c *ExecuteCommand) RunContext(args *ExecuteArgs) int {
 	// Plugin will match something like "packer-builder-amazon-ebs"
-	parts := pluginRegexp.FindStringSubmatch(args[0])
+	parts := pluginRegexp.FindStringSubmatch(args.CommandType)
 	if len(parts) != 3 {
 		c.Ui.Error(c.Help())
 		return 1
@@ -100,6 +127,10 @@ func (c *ExecuteCommand) Run(args []string) int {
 		return 1
 	}
 
+	if args.UseProtobuf {
+		server.UseProto = true
+	}
+
 	switch pluginType {
 	case "builder":
 		builder, found := Builders[pluginName]
@@ -138,11 +169,15 @@ func (c *ExecuteCommand) Run(args []string) int {
 
 func (*ExecuteCommand) Help() string {
 	helpText := `
-Usage: packer execute PLUGIN
+Usage: packer execute [options] PLUGIN
 
   Runs an internally-compiled version of a plugin from the packer binary.
 
   NOTE: this is an internal command and you should not call it yourself.
+
+Options:
+
+  --protobuf: use protobuf for serialising data over-the-wire instead of gob.
 `
 
 	return strings.TrimSpace(helpText)

command/validate.go

@@ -5,6 +5,7 @@ package command
 
 import (
 	"context"
+	"log"
 	"strings"
 
 	"github.com/hashicorp/packer/packer"
@@ -76,6 +77,10 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int
 		return ret
 	}
 
+	if packer.PackerUseProto {
+		log.Printf("[TRACE] Using protobuf for communication with plugins")
+	}
+
 	diags = packerStarter.Initialize(packer.InitializeOptions{
 		SkipDatasourcesExecution: !cla.EvaluateDatasources,
 		UseSequential:            cla.UseSequential,

config.go

@@ -16,10 +16,6 @@ import (
 	"github.com/hashicorp/packer/packer"
 )
 
-// PACKERSPACE is used to represent the spaces that separate args for a command
-// without being confused with spaces in the path to the command itself.
-const PACKERSPACE = "-PACKERSPACE-"
-
 type config struct {
 	DisableCheckpoint          bool              `json:"disable_checkpoint"`
 	DisableCheckpointSignature bool              `json:"disable_checkpoint_signature"`
@@ -109,43 +105,67 @@ func (c *config) discoverInternalComponents() error {
 	for builder := range command.Builders {
 		builder := builder
 		if !c.Plugins.Builders.Has(builder) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-builder-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, builder)
 			c.Plugins.Builders.Set(builder, func() (packersdk.Builder, error) {
-				return c.Plugins.Client(bin).Builder()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-builder-%s", builder))
+
+				return c.Plugins.Client(packerPath, args...).Builder()
 			})
 		}
 	}
 
 	for provisioner := range command.Provisioners {
 		provisioner := provisioner
 		if !c.Plugins.Provisioners.Has(provisioner) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-provisioner-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, provisioner)
 			c.Plugins.Provisioners.Set(provisioner, func() (packersdk.Provisioner, error) {
-				return c.Plugins.Client(bin).Provisioner()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-provisioner-%s", provisioner))
+
+				return c.Plugins.Client(packerPath, args...).Provisioner()
 			})
 		}
 	}
 
 	for postProcessor := range command.PostProcessors {
 		postProcessor := postProcessor
 		if !c.Plugins.PostProcessors.Has(postProcessor) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-post-processor-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, postProcessor)
 			c.Plugins.PostProcessors.Set(postProcessor, func() (packersdk.PostProcessor, error) {
-				return c.Plugins.Client(bin).PostProcessor()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-post-processor-%s", postProcessor))
+
+				return c.Plugins.Client(packerPath, args...).PostProcessor()
 			})
 		}
 	}
 
 	for dataSource := range command.Datasources {
 		dataSource := dataSource
 		if !c.Plugins.DataSources.Has(dataSource) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-datasource-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, dataSource)
 			c.Plugins.DataSources.Set(dataSource, func() (packersdk.Datasource, error) {
-				return c.Plugins.Client(bin).Datasource()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-datasource-%s", dataSource))
+
+				return c.Plugins.Client(packerPath, args...).Datasource()
 			})
 		}
 	}

go.mod

@@ -83,7 +83,7 @@ require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/aws/aws-sdk-go v1.44.114 // indirect
+	github.com/aws/aws-sdk-go v1.45.6 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/bgentry/speakeasy v0.2.0 // indirect
 	github.com/bmatcuk/doublestar v1.1.5 // indirect

go.sum

@@ -63,8 +63,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/aws/aws-sdk-go v1.44.114 h1:plIkWc/RsHr3DXBj4MEw9sEW4CcL/e2ryokc+CKyq1I=
-github.com/aws/aws-sdk-go v1.44.114/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.45.6 h1:Y2isQQBZsnO15dzUQo9YQRThtHgrV200XCH05BRHVJI=
+github.com/aws/aws-sdk-go v1.45.6/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -596,8 +596,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -639,20 +639,21 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -661,6 +662,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=