From a5bbec0671e3116d0588ff878f3ee4fe076a88d2 Mon Sep 17 00:00:00 2001 From: hamedsalim1999 Date: Mon, 1 Jan 2024 15:34:31 +0100 Subject: [PATCH 01/25] Add url parameter for Plugin --- config/types.go | 1 + plugin/plugin_registry.go | 1 + 2 files changed, 2 insertions(+) diff --git a/config/types.go b/config/types.go index 7cbeb245..e3364284 100644 --- a/config/types.go +++ b/config/types.go @@ -12,6 +12,7 @@ type Plugin struct { Args []string `json:"args"` Env []string `json:"env" jsonschema:"required"` Checksum string `json:"checksum" jsonschema:"required"` + URL string `json:"url"` } type PluginConfig struct { diff --git a/plugin/plugin_registry.go b/plugin/plugin_registry.go index ff4127e6..a30b7c98 100644 --- a/plugin/plugin_registry.go +++ b/plugin/plugin_registry.go @@ -398,6 +398,7 @@ func (reg *Registry) LoadPlugins( pluginCtx, span := otel.Tracer("").Start(ctx, "Load plugin") span.SetAttributes(attribute.Int("priority", priority)) span.SetAttributes(attribute.String("name", pCfg.Name)) + span.SetAttributes(attribute.String("url", pCfg.URL)) span.SetAttributes(attribute.Bool("enabled", pCfg.Enabled)) span.SetAttributes(attribute.String("checksum", pCfg.Checksum)) span.SetAttributes(attribute.String("local_path", pCfg.LocalPath)) From 2d422ac4228127f5b1190d720043999d3dc446cc Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 1 Jan 2024 19:42:17 +0100 Subject: [PATCH 02/25] Move plugin install logic to utils Add ability to install plugin using url parameter in plugin config (plugins) --- cmd/plugin_install.go | 386 ++-------------------------------------- cmd/utils.go | 402 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 4 + 4 files changed, 426 insertions(+), 367 deletions(-) diff --git a/cmd/plugin_install.go b/cmd/plugin_install.go index 19e1bf50..4d739b80 100644 --- a/cmd/plugin_install.go +++ b/cmd/plugin_install.go @@ -1,20 +1,12 @@ package cmd import ( - "context" - "fmt" "log" "os" - "path/filepath" - "regexp" - "runtime" - "slices" - "strings" - "github.com/codingsince1985/checksum" "github.com/gatewayd-io/gatewayd/config" "github.com/getsentry/sentry-go" - "github.com/google/go-github/v53/github" + "github.com/spf13/cast" "github.com/spf13/cobra" yamlv3 "gopkg.in/yaml.v3" ) @@ -45,9 +37,6 @@ var pluginInstallCmd = &cobra.Command{ Short: "Install a plugin from a local archive or a GitHub repository", Example: " gatewayd plugin install github.com/gatewayd-io/gatewayd-plugin-cache@latest", Run: func(cmd *cobra.Command, args []string) { - // This is a list of files that will be deleted after the plugin is installed. - toBeDeleted := []string{} - // Enable Sentry. if enableSentry { // Initialize Sentry. @@ -67,199 +56,6 @@ var pluginInstallCmd = &cobra.Command{ defer sentry.Recover() } - // Validate the number of arguments. - if len(args) < 1 { - cmd.Println( - "Invalid URL. Use the following format: github.com/account/repository@version") - return - } - - var releaseID int64 - var downloadURL string - var pluginFilename string - var pluginName string - var err error - var checksumsFilename string - var client *github.Client - var account string - - // Strip scheme from the plugin URL. - args[0] = strings.TrimPrefix(args[0], "http://") - args[0] = strings.TrimPrefix(args[0], "https://") - - if !strings.HasPrefix(args[0], GitHubURLPrefix) { - // Pull the plugin from a local archive. - pluginFilename = filepath.Clean(args[0]) - if _, err := os.Stat(pluginFilename); os.IsNotExist(err) { - cmd.Println("The plugin file could not be found") - return - } - } - - // Validate the URL. - validGitHubURL := regexp.MustCompile(GitHubURLRegex) - if !validGitHubURL.MatchString(args[0]) { - cmd.Println( - "Invalid URL. Use the following format: github.com/account/repository@version") - return - } - - // Get the plugin version. - pluginVersion := LatestVersion - splittedURL := strings.Split(args[0], "@") - // If the version is not specified, use the latest version. - if len(splittedURL) < NumParts { - cmd.Println("Version not specified. Using latest version") - } - if len(splittedURL) >= NumParts { - pluginVersion = splittedURL[1] - } - - // Get the plugin account and repository. - accountRepo := strings.Split(strings.TrimPrefix(splittedURL[0], GitHubURLPrefix), "/") - if len(accountRepo) != NumParts { - cmd.Println( - "Invalid URL. Use the following format: github.com/account/repository@version") - return - } - account = accountRepo[0] - pluginName = accountRepo[1] - if account == "" || pluginName == "" { - cmd.Println( - "Invalid URL. Use the following format: github.com/account/repository@version") - return - } - - // Get the release artifact from GitHub. - client = github.NewClient(nil) - var release *github.RepositoryRelease - - if pluginVersion == LatestVersion || pluginVersion == "" { - // Get the latest release. - release, _, err = client.Repositories.GetLatestRelease( - context.Background(), account, pluginName) - } else if strings.HasPrefix(pluginVersion, "v") { - // Get an specific release. - release, _, err = client.Repositories.GetReleaseByTag( - context.Background(), account, pluginName, pluginVersion) - } - - if err != nil { - cmd.Println("The plugin could not be found: ", err.Error()) - return - } - - if release == nil { - cmd.Println("The plugin could not be found in the release assets") - return - } - - // Get the archive extension. - archiveExt := ExtOthers - if runtime.GOOS == "windows" { - archiveExt = ExtWindows - } - - // Find and download the plugin binary from the release assets. - pluginFilename, downloadURL, releaseID = findAsset(release, func(name string) bool { - return strings.Contains(name, runtime.GOOS) && - strings.Contains(name, runtime.GOARCH) && - strings.Contains(name, archiveExt) - }) - - var filePath string - if downloadURL != "" && releaseID != 0 { - cmd.Println("Downloading", downloadURL) - filePath, err = downloadFile(client, account, pluginName, releaseID, pluginFilename) - toBeDeleted = append(toBeDeleted, filePath) - if err != nil { - cmd.Println("Download failed: ", err) - if cleanup { - deleteFiles(toBeDeleted) - } - return - } - cmd.Println("Download completed successfully") - } else { - cmd.Println("The plugin file could not be found in the release assets") - return - } - - // Find and download the checksums.txt from the release assets. - checksumsFilename, downloadURL, releaseID = findAsset(release, func(name string) bool { - return strings.Contains(name, "checksums.txt") - }) - if checksumsFilename != "" && downloadURL != "" && releaseID != 0 { - cmd.Println("Downloading", downloadURL) - filePath, err = downloadFile(client, account, pluginName, releaseID, checksumsFilename) - toBeDeleted = append(toBeDeleted, filePath) - if err != nil { - cmd.Println("Download failed: ", err) - if cleanup { - deleteFiles(toBeDeleted) - } - return - } - cmd.Println("Download completed successfully") - } else { - cmd.Println("The checksum file could not be found in the release assets") - return - } - - // Read the checksums text file. - checksums, err := os.ReadFile(checksumsFilename) - if err != nil { - cmd.Println("There was an error reading the checksums file: ", err) - return - } - - // Get the checksum for the plugin binary. - sum, err := checksum.SHA256sum(pluginFilename) - if err != nil { - cmd.Println("There was an error calculating the checksum: ", err) - return - } - - // Verify the checksums. - checksumLines := strings.Split(string(checksums), "\n") - for _, line := range checksumLines { - if strings.Contains(line, pluginFilename) { - checksum := strings.Split(line, " ")[0] - if checksum != sum { - cmd.Println("Checksum verification failed") - return - } - - cmd.Println("Checksum verification passed") - break - } - } - - if pullOnly { - cmd.Println("Plugin binary downloaded to", pluginFilename) - // Only the checksums file will be deleted if the --pull-only flag is set. - if err := os.Remove(checksumsFilename); err != nil { - cmd.Println("There was an error deleting the file: ", err) - } - return - } - - // Create a new gatewayd_plugins.yaml file if it doesn't exist. - if _, err := os.Stat(pluginConfigFile); os.IsNotExist(err) { - generateConfig(cmd, Plugins, pluginConfigFile, false) - } else { - // If the config file exists, we should prompt the user to backup - // the plugins configuration file. - if !backupConfig && !noPrompt { - cmd.Print("Do you want to backup the plugins configuration file? [Y/n] ") - var backupOption string - _, err := fmt.Scanln(&backupOption) - if err == nil && (backupOption == "y" || backupOption == "Y") { - backupConfig = true - } - } - } - // Read the gatewayd_plugins.yaml file. pluginsConfig, err := os.ReadFile(pluginConfigFile) if err != nil { @@ -279,180 +75,36 @@ var pluginInstallCmd = &cobra.Command{ return } - // Check if the plugin is already installed. + // Get the list of plugin download URLs. + var pluginURLs []string for _, plugin := range pluginsList { - // User already chosen to update the plugin using the --update CLI flag. - if update { - break - } - if pluginInstance, ok := plugin.(map[string]interface{}); ok { - if pluginInstance["name"] == pluginName { - // Show a list of options to the user. - cmd.Println("Plugin is already installed.") - if !noPrompt { - cmd.Print("Do you want to update the plugin? [y/N] ") - - var updateOption string - _, err := fmt.Scanln(&updateOption) - if err == nil && (updateOption == "y" || updateOption == "Y") { - break - } - } - - cmd.Println("Aborting...") - if cleanup { - deleteFiles(toBeDeleted) - } - return + // Get the plugin URL. + url := cast.ToString(pluginInstance["url"]) + if url != "" { + pluginURLs = append(pluginURLs, url) } } } - // Check if the user wants to take a backup of the plugins configuration file. - if backupConfig { - backupFilename := fmt.Sprintf("%s.bak", pluginConfigFile) - if err := os.WriteFile(backupFilename, pluginsConfig, FilePermissions); err != nil { - cmd.Println("There was an error backing up the plugins configuration file: ", err) - } - cmd.Println("Backup completed successfully") - } - - // Extract the archive. - var filenames []string - if runtime.GOOS == "windows" { - filenames, err = extractZip(pluginFilename, pluginOutputDir) - } else { - filenames, err = extractTarGz(pluginFilename, pluginOutputDir) - } - - if err != nil { - cmd.Println("There was an error extracting the plugin archive: ", err) - if cleanup { - deleteFiles(toBeDeleted) - } + // Validate the number of arguments. + if len(args) < 1 && len(pluginURLs) < 1 { + cmd.Println( + "Invalid URL. Use the following format: github.com/account/repository@version") return } - // Delete all the files except the extracted plugin binary, - // which will be deleted from the list further down. - toBeDeleted = append(toBeDeleted, filenames...) - - // Find the extracted plugin binary. - localPath := "" - pluginFileSum := "" - for _, filename := range filenames { - if strings.Contains(filename, pluginName) { - cmd.Println("Plugin binary extracted to", filename) - - // Remove the plugin binary from the list of files to be deleted. - toBeDeleted = slices.DeleteFunc[[]string, string](toBeDeleted, func(s string) bool { - return s == filename - }) - - localPath = filename - // Get the checksum for the extracted plugin binary. - // TODO: Should we verify the checksum using the checksum.txt file instead? - pluginFileSum, err = checksum.SHA256sum(filename) - if err != nil { - cmd.Println("There was an error calculating the checksum: ", err) - return - } - break - } - } - - var contents string - if strings.HasPrefix(args[0], GitHubURLPrefix) { - // Get the list of files in the repository. - var repoContents *github.RepositoryContent - repoContents, _, _, err = client.Repositories.GetContents( - context.Background(), account, pluginName, DefaultPluginConfigFilename, nil) - if err != nil { - cmd.Println( - "There was an error getting the default plugins configuration file: ", err) - return - } - // Get the contents of the file. - contents, err = repoContents.GetContent() - if err != nil { - cmd.Println( - "There was an error getting the default plugins configuration file: ", err) - return - } + if len(args) > 0 { + // Install the plugin from the CLI argument. + cmd.Println("Installing plugin from CLI argument") + installPlugin(cmd, args[0]) } else { - // Get the contents of the file. - contentsBytes, err := os.ReadFile( - filepath.Join(pluginOutputDir, DefaultPluginConfigFilename)) - if err != nil { - cmd.Println( - "There was an error getting the default plugins configuration file: ", err) - return - } - contents = string(contentsBytes) - } - - // Get the plugin configuration from the downloaded plugins configuration file. - var downloadedPluginConfig map[string]interface{} - if err := yamlv3.Unmarshal([]byte(contents), &downloadedPluginConfig); err != nil { - cmd.Println("Failed to unmarshal the downloaded plugins configuration file: ", err) - return - } - defaultPluginConfig, ok := downloadedPluginConfig["plugins"].([]interface{}) - if !ok { - cmd.Println("There was an error reading the plugins file from the repository") - return - } - // Get the plugin configuration. - pluginConfig, ok := defaultPluginConfig[0].(map[string]interface{}) - if !ok { - cmd.Println("There was an error reading the default plugin configuration") - return - } - - // Update the plugin's local path and checksum. - pluginConfig["localPath"] = localPath - pluginConfig["checksum"] = pluginFileSum - - // Add the plugin config to the list of plugin configs. - added := false - for idx, plugin := range pluginsList { - if pluginInstance, ok := plugin.(map[string]interface{}); ok { - if pluginInstance["name"] == pluginName { - pluginsList[idx] = pluginConfig - added = true - break - } + // Install all the plugins from the plugins configuration file. + cmd.Println("Installing plugins from plugins configuration file") + for _, pluginURL := range pluginURLs { + installPlugin(cmd, pluginURL) } } - if !added { - pluginsList = append(pluginsList, pluginConfig) - } - - // Merge the result back into the config map. - localPluginsConfig["plugins"] = pluginsList - - // Marshal the map into YAML. - updatedPlugins, err := yamlv3.Marshal(localPluginsConfig) - if err != nil { - cmd.Println("There was an error marshalling the plugins configuration: ", err) - return - } - - // Write the YAML to the plugins config file. - if err = os.WriteFile(pluginConfigFile, updatedPlugins, FilePermissions); err != nil { - cmd.Println("There was an error writing the plugins configuration file: ", err) - return - } - - // Delete the downloaded and extracted files, except the plugin binary, - // if the --cleanup flag is set. - if cleanup { - deleteFiles(toBeDeleted) - } - - // TODO: Add a rollback mechanism. - cmd.Println("Plugin installed successfully") }, } diff --git a/cmd/utils.go b/cmd/utils.go index c3115d83..aee95570 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -14,8 +14,12 @@ import ( "os" "path" "path/filepath" + "regexp" + "runtime" + "slices" "strings" + "github.com/codingsince1985/checksum" "github.com/gatewayd-io/gatewayd/config" gerr "github.com/gatewayd-io/gatewayd/errors" "github.com/google/go-github/v53/github" @@ -25,6 +29,7 @@ import ( "github.com/knadh/koanf/parsers/yaml" jsonSchemaV5 "github.com/santhosh-tekuri/jsonschema/v5" "github.com/spf13/cobra" + yamlv3 "gopkg.in/yaml.v3" ) type ( @@ -442,3 +447,400 @@ func deleteFiles(toBeDeleted []string) { } } } + +// installPlugin installs a plugin from a given URL. +func installPlugin(cmd *cobra.Command, pluginURL string) { + // This is a list of files that will be deleted after the plugin is installed. + toBeDeleted := []string{} + + var ( + releaseID int64 + downloadURL string + pluginFilename string + pluginName string + err error + checksumsFilename string + client *github.Client + account string + ) + + // Strip scheme from the plugin URL. + pluginURL = strings.TrimPrefix(pluginURL, "http://") + pluginURL = strings.TrimPrefix(pluginURL, "https://") + + if !strings.HasPrefix(pluginURL, GitHubURLPrefix) { + // Pull the plugin from a local archive. + pluginFilename = filepath.Clean(pluginURL) + if _, err := os.Stat(pluginFilename); os.IsNotExist(err) { + cmd.Println("The plugin file could not be found") + return + } + } + + // Validate the URL. + splittedURL := strings.Split(pluginURL, "@") + if len(splittedURL) < NumParts { + if pluginFilename == "" { + // If the version is not specified, use the latest version. + pluginURL = fmt.Sprintf("%s@%s", pluginURL, LatestVersion) + } + } + + validGitHubURL := regexp.MustCompile(GitHubURLRegex) + if !validGitHubURL.MatchString(pluginURL) { + cmd.Println( + "Invalid URL. Use the following format: github.com/account/repository@version") + return + } + + // Get the plugin version. + pluginVersion := LatestVersion + splittedURL = strings.Split(pluginURL, "@") + // If the version is not specified, use the latest version. + if len(splittedURL) < NumParts { + cmd.Println("Version not specified. Using latest version") + } + if len(splittedURL) >= NumParts { + pluginVersion = splittedURL[1] + } + + // Get the plugin account and repository. + accountRepo := strings.Split(strings.TrimPrefix(splittedURL[0], GitHubURLPrefix), "/") + if len(accountRepo) != NumParts { + cmd.Println( + "Invalid URL. Use the following format: github.com/account/repository@version") + return + } + account = accountRepo[0] + pluginName = accountRepo[1] + if account == "" || pluginName == "" { + cmd.Println( + "Invalid URL. Use the following format: github.com/account/repository@version") + return + } + + // Get the release artifact from GitHub. + client = github.NewClient(nil) + var release *github.RepositoryRelease + + if pluginVersion == LatestVersion || pluginVersion == "" { + // Get the latest release. + release, _, err = client.Repositories.GetLatestRelease( + context.Background(), account, pluginName) + } else if strings.HasPrefix(pluginVersion, "v") { + // Get an specific release. + release, _, err = client.Repositories.GetReleaseByTag( + context.Background(), account, pluginName, pluginVersion) + } + + if err != nil { + cmd.Println("The plugin could not be found: ", err.Error()) + return + } + + if release == nil { + cmd.Println("The plugin could not be found in the release assets") + return + } + + // Get the archive extension. + archiveExt := ExtOthers + if runtime.GOOS == "windows" { + archiveExt = ExtWindows + } + + // Find and download the plugin binary from the release assets. + pluginFilename, downloadURL, releaseID = findAsset(release, func(name string) bool { + return strings.Contains(name, runtime.GOOS) && + strings.Contains(name, runtime.GOARCH) && + strings.Contains(name, archiveExt) + }) + + var filePath string + if downloadURL != "" && releaseID != 0 { + cmd.Println("Downloading", downloadURL) + filePath, err = downloadFile(client, account, pluginName, releaseID, pluginFilename) + toBeDeleted = append(toBeDeleted, filePath) + if err != nil { + cmd.Println("Download failed: ", err) + if cleanup { + deleteFiles(toBeDeleted) + } + return + } + cmd.Println("Download completed successfully") + } else { + cmd.Println("The plugin file could not be found in the release assets") + return + } + + // Find and download the checksums.txt from the release assets. + checksumsFilename, downloadURL, releaseID = findAsset(release, func(name string) bool { + return strings.Contains(name, "checksums.txt") + }) + if checksumsFilename != "" && downloadURL != "" && releaseID != 0 { + cmd.Println("Downloading", downloadURL) + filePath, err = downloadFile(client, account, pluginName, releaseID, checksumsFilename) + toBeDeleted = append(toBeDeleted, filePath) + if err != nil { + cmd.Println("Download failed: ", err) + if cleanup { + deleteFiles(toBeDeleted) + } + return + } + cmd.Println("Download completed successfully") + } else { + cmd.Println("The checksum file could not be found in the release assets") + return + } + + // Read the checksums text file. + checksums, err := os.ReadFile(checksumsFilename) + if err != nil { + cmd.Println("There was an error reading the checksums file: ", err) + return + } + + // Get the checksum for the plugin binary. + sum, err := checksum.SHA256sum(pluginFilename) + if err != nil { + cmd.Println("There was an error calculating the checksum: ", err) + return + } + + // Verify the checksums. + checksumLines := strings.Split(string(checksums), "\n") + for _, line := range checksumLines { + if strings.Contains(line, pluginFilename) { + checksum := strings.Split(line, " ")[0] + if checksum != sum { + cmd.Println("Checksum verification failed") + return + } + + cmd.Println("Checksum verification passed") + break + } + } + + if pullOnly { + cmd.Println("Plugin binary downloaded to", pluginFilename) + // Only the checksums file will be deleted if the --pull-only flag is set. + if err := os.Remove(checksumsFilename); err != nil { + cmd.Println("There was an error deleting the file: ", err) + } + return + } + + // Create a new gatewayd_plugins.yaml file if it doesn't exist. + if _, err := os.Stat(pluginConfigFile); os.IsNotExist(err) { + generateConfig(cmd, Plugins, pluginConfigFile, false) + } else { + // If the config file exists, we should prompt the user to backup + // the plugins configuration file. + if !backupConfig && !noPrompt { + cmd.Print("Do you want to backup the plugins configuration file? [Y/n] ") + var backupOption string + _, err := fmt.Scanln(&backupOption) + if err == nil && (backupOption == "y" || backupOption == "Y") { + backupConfig = true + } + } + } + + // Read the gatewayd_plugins.yaml file. + pluginsConfig, err := os.ReadFile(pluginConfigFile) + if err != nil { + log.Println(err) + return + } + + // Get the registered plugins from the plugins configuration file. + var localPluginsConfig map[string]interface{} + if err := yamlv3.Unmarshal(pluginsConfig, &localPluginsConfig); err != nil { + log.Println("Failed to unmarshal the plugins configuration file: ", err) + return + } + pluginsList, ok := localPluginsConfig["plugins"].([]interface{}) //nolint:varnamelen + if !ok { + log.Println("There was an error reading the plugins file from disk") + return + } + + // Check if the plugin is already installed. + for _, plugin := range pluginsList { + // User already chosen to update the plugin using the --update CLI flag. + if update { + break + } + + //nolint:nestif + if pluginInstance, ok := plugin.(map[string]interface{}); ok { + if pluginInstance["name"] == pluginName { + // Show a list of options to the user. + cmd.Println("Plugin is already installed.") + if !noPrompt { + cmd.Print("Do you want to update the plugin? [y/N] ") + + var updateOption string + _, err := fmt.Scanln(&updateOption) + if err == nil && (updateOption == "y" || updateOption == "Y") { + break + } + } + + cmd.Println("Aborting...") + if cleanup { + deleteFiles(toBeDeleted) + } + return + } + } + } + + // Check if the user wants to take a backup of the plugins configuration file. + if backupConfig { + backupFilename := fmt.Sprintf("%s.bak", pluginConfigFile) + if err := os.WriteFile(backupFilename, pluginsConfig, FilePermissions); err != nil { + cmd.Println("There was an error backing up the plugins configuration file: ", err) + } + cmd.Println("Backup completed successfully") + } + + // Extract the archive. + var filenames []string + if runtime.GOOS == "windows" { + filenames, err = extractZip(pluginFilename, pluginOutputDir) + } else { + filenames, err = extractTarGz(pluginFilename, pluginOutputDir) + } + + if err != nil { + cmd.Println("There was an error extracting the plugin archive: ", err) + if cleanup { + deleteFiles(toBeDeleted) + } + return + } + + // Delete all the files except the extracted plugin binary, + // which will be deleted from the list further down. + toBeDeleted = append(toBeDeleted, filenames...) + + // Find the extracted plugin binary. + localPath := "" + pluginFileSum := "" + for _, filename := range filenames { + if strings.Contains(filename, pluginName) { + cmd.Println("Plugin binary extracted to", filename) + + // Remove the plugin binary from the list of files to be deleted. + toBeDeleted = slices.DeleteFunc[[]string, string](toBeDeleted, func(s string) bool { + return s == filename + }) + + localPath = filename + // Get the checksum for the extracted plugin binary. + // TODO: Should we verify the checksum using the checksum.txt file instead? + pluginFileSum, err = checksum.SHA256sum(filename) + if err != nil { + cmd.Println("There was an error calculating the checksum: ", err) + return + } + break + } + } + + var contents string + if strings.HasPrefix(pluginURL, GitHubURLPrefix) { + // Get the list of files in the repository. + var repoContents *github.RepositoryContent + repoContents, _, _, err = client.Repositories.GetContents( + context.Background(), account, pluginName, DefaultPluginConfigFilename, nil) + if err != nil { + cmd.Println( + "There was an error getting the default plugins configuration file: ", err) + return + } + // Get the contents of the file. + contents, err = repoContents.GetContent() + if err != nil { + cmd.Println( + "There was an error getting the default plugins configuration file: ", err) + return + } + } else { + // Get the contents of the file. + contentsBytes, err := os.ReadFile( + filepath.Join(pluginOutputDir, DefaultPluginConfigFilename)) + if err != nil { + cmd.Println( + "There was an error getting the default plugins configuration file: ", err) + return + } + contents = string(contentsBytes) + } + + // Get the plugin configuration from the downloaded plugins configuration file. + var downloadedPluginConfig map[string]interface{} + if err := yamlv3.Unmarshal([]byte(contents), &downloadedPluginConfig); err != nil { + cmd.Println("Failed to unmarshal the downloaded plugins configuration file: ", err) + return + } + defaultPluginConfig, ok := downloadedPluginConfig["plugins"].([]interface{}) + if !ok { + cmd.Println("There was an error reading the plugins file from the repository") + return + } + // Get the plugin configuration. + pluginConfig, ok := defaultPluginConfig[0].(map[string]interface{}) + if !ok { + cmd.Println("There was an error reading the default plugin configuration") + return + } + + // Update the plugin's local path and checksum. + pluginConfig["localPath"] = localPath + pluginConfig["checksum"] = pluginFileSum + + // Add the plugin config to the list of plugin configs. + added := false + for idx, plugin := range pluginsList { + if pluginInstance, ok := plugin.(map[string]interface{}); ok { + if pluginInstance["name"] == pluginName { + pluginsList[idx] = pluginConfig + added = true + break + } + } + } + if !added { + pluginsList = append(pluginsList, pluginConfig) + } + + // Merge the result back into the config map. + localPluginsConfig["plugins"] = pluginsList + + // Marshal the map into YAML. + updatedPlugins, err := yamlv3.Marshal(localPluginsConfig) + if err != nil { + cmd.Println("There was an error marshalling the plugins configuration: ", err) + return + } + + // Write the YAML to the plugins config file. + if err = os.WriteFile(pluginConfigFile, updatedPlugins, FilePermissions); err != nil { + cmd.Println("There was an error writing the plugins configuration file: ", err) + return + } + + // Delete the downloaded and extracted files, except the plugin binary, + // if the --cleanup flag is set. + if cleanup { + deleteFiles(toBeDeleted) + } + + // TODO: Add a rollback mechanism. + cmd.Println("Plugin installed successfully") +} diff --git a/go.mod b/go.mod index 95e462c5..d1f14b07 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/prometheus/common v0.46.0 github.com/rs/zerolog v1.31.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 + github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04 diff --git a/go.sum b/go.sum index 3181f5e2..b4c7bd67 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= @@ -338,6 +340,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= From 093a3f9cc45d3a010c2b666ea62a9a84f4054ba9 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 1 Jan 2024 19:45:49 +0100 Subject: [PATCH 03/25] Add url of the cache plugin --- gatewayd_plugins.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/gatewayd_plugins.yaml b/gatewayd_plugins.yaml index 33bb7374..5beaa108 100644 --- a/gatewayd_plugins.yaml +++ b/gatewayd_plugins.yaml @@ -79,6 +79,7 @@ startTimeout: 1m plugins: - name: gatewayd-plugin-cache enabled: True + url: github.com/gatewayd/gatewayd-plugin-cache@latest localPath: ../gatewayd-plugin-cache/gatewayd-plugin-cache args: ["--log-level", "debug"] env: From decddd0f8b0b4023b924bef6131ccdb8cb1903d7 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Wed, 24 Jan 2024 22:26:35 +0100 Subject: [PATCH 04/25] Make plugin install work with CLI args and URLs in the gatewayd_plugins.yaml --- .golangci.yaml | 1 + cmd/plugin_install.go | 91 ++++++----- cmd/utils.go | 348 ++++++++++++++++++++++++------------------ gatewayd_plugins.yaml | 2 +- 4 files changed, 252 insertions(+), 190 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 6095aa3e..bebf047b 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -46,6 +46,7 @@ linters-settings: - "github.com/knadh/koanf" - "github.com/panjf2000/gnet/v2" - "github.com/spf13/cobra" + - "github.com/spf13/cast" - "github.com/invopop/jsonschema" - "github.com/santhosh-tekuri/jsonschema/v5" - "github.com/NYTimes/gziphandler" diff --git a/cmd/plugin_install.go b/cmd/plugin_install.go index 4d739b80..61ddcab6 100644 --- a/cmd/plugin_install.go +++ b/cmd/plugin_install.go @@ -1,7 +1,6 @@ package cmd import ( - "log" "os" "github.com/gatewayd-io/gatewayd/config" @@ -11,6 +10,12 @@ import ( yamlv3 "gopkg.in/yaml.v3" ) +type ( + Location string + Source string + Extension string +) + const ( NumParts int = 2 LatestVersion string = "latest" @@ -18,8 +23,13 @@ const ( DefaultPluginConfigFilename string = "./gatewayd_plugin.yaml" GitHubURLPrefix string = "github.com/" GitHubURLRegex string = `^github.com\/[a-zA-Z0-9\-]+\/[a-zA-Z0-9\-]+@(?:latest|v(=|>=|<=|=>|=<|>|<|!=|~|~>|\^)?(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$` //nolint:lll - ExtWindows string = ".zip" - ExtOthers string = ".tar.gz" + LocationArgs Location = "args" + LocationConfig Location = "config" + SourceUnknown Source = "unknown" + SourceFile Source = "file" + SourceGitHub Source = "github" + ExtensionZip Extension = ".zip" + ExtensionTarGz Extension = ".tar.gz" ) var ( @@ -29,6 +39,7 @@ var ( update bool backupConfig bool noPrompt bool + pluginName string ) // pluginInstallCmd represents the plugin install command. @@ -56,54 +67,58 @@ var pluginInstallCmd = &cobra.Command{ defer sentry.Recover() } - // Read the gatewayd_plugins.yaml file. - pluginsConfig, err := os.ReadFile(pluginConfigFile) - if err != nil { - log.Println(err) - return - } + switch detectInstallLocation(args) { + case LocationArgs: + // Install the plugin from the CLI argument. + cmd.Println("Installing plugin from CLI argument") + installPlugin(cmd, args[0]) + case LocationConfig: + // Read the gatewayd_plugins.yaml file. + pluginsConfig, err := os.ReadFile(pluginConfigFile) + if err != nil { + cmd.Println(err) + return + } - // Get the registered plugins from the plugins configuration file. - var localPluginsConfig map[string]interface{} - if err := yamlv3.Unmarshal(pluginsConfig, &localPluginsConfig); err != nil { - log.Println("Failed to unmarshal the plugins configuration file: ", err) - return - } - pluginsList, ok := localPluginsConfig["plugins"].([]interface{}) //nolint:varnamelen - if !ok { - log.Println("There was an error reading the plugins file from disk") - return - } + // Get the registered plugins from the plugins configuration file. + var localPluginsConfig map[string]interface{} + if err := yamlv3.Unmarshal(pluginsConfig, &localPluginsConfig); err != nil { + cmd.Println("Failed to unmarshal the plugins configuration file: ", err) + return + } + pluginsList := cast.ToSlice(localPluginsConfig["plugins"]) + + // Get the list of plugin download URLs. + var pluginURLs []string + for _, plugin := range pluginsList { + // Get the plugin instance. + pluginInstance := cast.ToStringMapString(plugin) - // Get the list of plugin download URLs. - var pluginURLs []string - for _, plugin := range pluginsList { - if pluginInstance, ok := plugin.(map[string]interface{}); ok { - // Get the plugin URL. + // Append the plugin URL to the list of plugin URLs. + name := cast.ToString(pluginInstance["name"]) url := cast.ToString(pluginInstance["url"]) if url != "" { pluginURLs = append(pluginURLs, url) + } else { + cmd.Println("Plugin URL or file path not found in the plugins configuration file for", name) + return } } - } - // Validate the number of arguments. - if len(args) < 1 && len(pluginURLs) < 1 { - cmd.Println( - "Invalid URL. Use the following format: github.com/account/repository@version") - return - } + // Validate the plugin URLs. + if len(args) == 0 && len(pluginURLs) == 0 { + cmd.Println( + "No plugin URLs or file path found in the plugins configuration file or CLI argument") + return + } - if len(args) > 0 { - // Install the plugin from the CLI argument. - cmd.Println("Installing plugin from CLI argument") - installPlugin(cmd, args[0]) - } else { // Install all the plugins from the plugins configuration file. cmd.Println("Installing plugins from plugins configuration file") for _, pluginURL := range pluginURLs { installPlugin(cmd, pluginURL) } + default: + cmd.Println("Invalid plugin URL or file path") } }, } @@ -128,6 +143,8 @@ func init() { &update, "update", false, "Update the plugin if it already exists") pluginInstallCmd.Flags().BoolVar( &backupConfig, "backup", false, "Backup the plugins configuration file before installing the plugin") + pluginInstallCmd.Flags().StringVarP( + &pluginName, "name", "n", "", "Name of the plugin") pluginInstallCmd.Flags().BoolVar( &enableSentry, "sentry", true, "Enable Sentry") // Already exists in run.go } diff --git a/cmd/utils.go b/cmd/utils.go index aee95570..760a4b61 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -442,198 +442,238 @@ func downloadFile( func deleteFiles(toBeDeleted []string) { for _, filename := range toBeDeleted { if err := os.Remove(filename); err != nil { - log.Println("There was an error deleting the file: ", err) + fmt.Println("There was an error deleting the file: ", err) return } } } +// detectInstallLocation detects the install location based on the number of arguments. +func detectInstallLocation(args []string) Location { + if len(args) == 0 { + return LocationConfig + } + + return LocationArgs +} + +// detectSource detects the source of the path. +func detectSource(path string) Source { + if _, err := os.Stat(path); err == nil { + return SourceFile + } + + // Check if the path is a URL. + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") || strings.HasPrefix(path, GitHubURLPrefix) { //nolint:lll + return SourceGitHub + } + + return SourceUnknown +} + +// getFileExtension returns the extension of the archive based on the OS. +func getFileExtension() Extension { + if runtime.GOOS == "windows" { + return ExtensionZip + } + + return ExtensionTarGz +} + // installPlugin installs a plugin from a given URL. func installPlugin(cmd *cobra.Command, pluginURL string) { - // This is a list of files that will be deleted after the plugin is installed. - toBeDeleted := []string{} - var ( + // This is a list of files that will be deleted after the plugin is installed. + toBeDeleted []string = []string{} + + // Source of the plugin: file or GitHub. + source Source = detectSource(pluginURL) + + // The extension of the archive based on the OS: .zip or .tar.gz. + archiveExt Extension = getFileExtension() + releaseID int64 downloadURL string pluginFilename string - pluginName string - err error checksumsFilename string - client *github.Client account string + err error + client *github.Client ) - // Strip scheme from the plugin URL. - pluginURL = strings.TrimPrefix(pluginURL, "http://") - pluginURL = strings.TrimPrefix(pluginURL, "https://") - - if !strings.HasPrefix(pluginURL, GitHubURLPrefix) { + switch source { + case SourceFile: // Pull the plugin from a local archive. pluginFilename = filepath.Clean(pluginURL) if _, err := os.Stat(pluginFilename); os.IsNotExist(err) { cmd.Println("The plugin file could not be found") return } - } - // Validate the URL. - splittedURL := strings.Split(pluginURL, "@") - if len(splittedURL) < NumParts { - if pluginFilename == "" { - // If the version is not specified, use the latest version. - pluginURL = fmt.Sprintf("%s@%s", pluginURL, LatestVersion) + if pluginName == "" { + cmd.Println("Plugin name not specified") + return + } + case SourceGitHub: + // Strip scheme from the plugin URL. + pluginURL = strings.TrimPrefix(strings.TrimPrefix(pluginURL, "http://"), "https://") + + // Validate the URL. + splittedURL := strings.Split(pluginURL, "@") + if len(splittedURL) < NumParts { + if pluginFilename == "" { + // If the version is not specified, use the latest version. + pluginURL = fmt.Sprintf("%s@%s", pluginURL, LatestVersion) + } } - } - - validGitHubURL := regexp.MustCompile(GitHubURLRegex) - if !validGitHubURL.MatchString(pluginURL) { - cmd.Println( - "Invalid URL. Use the following format: github.com/account/repository@version") - return - } - - // Get the plugin version. - pluginVersion := LatestVersion - splittedURL = strings.Split(pluginURL, "@") - // If the version is not specified, use the latest version. - if len(splittedURL) < NumParts { - cmd.Println("Version not specified. Using latest version") - } - if len(splittedURL) >= NumParts { - pluginVersion = splittedURL[1] - } - // Get the plugin account and repository. - accountRepo := strings.Split(strings.TrimPrefix(splittedURL[0], GitHubURLPrefix), "/") - if len(accountRepo) != NumParts { - cmd.Println( - "Invalid URL. Use the following format: github.com/account/repository@version") - return - } - account = accountRepo[0] - pluginName = accountRepo[1] - if account == "" || pluginName == "" { - cmd.Println( - "Invalid URL. Use the following format: github.com/account/repository@version") - return - } + validGitHubURL := regexp.MustCompile(GitHubURLRegex) + if !validGitHubURL.MatchString(pluginURL) { + cmd.Println( + "Invalid URL. Use the following format: github.com/account/repository@version") + return + } - // Get the release artifact from GitHub. - client = github.NewClient(nil) - var release *github.RepositoryRelease + // Get the plugin version. + pluginVersion := LatestVersion + splittedURL = strings.Split(pluginURL, "@") + // If the version is not specified, use the latest version. + if len(splittedURL) < NumParts { + cmd.Println("Version not specified. Using latest version") + } + if len(splittedURL) >= NumParts { + pluginVersion = splittedURL[1] + } - if pluginVersion == LatestVersion || pluginVersion == "" { - // Get the latest release. - release, _, err = client.Repositories.GetLatestRelease( - context.Background(), account, pluginName) - } else if strings.HasPrefix(pluginVersion, "v") { - // Get an specific release. - release, _, err = client.Repositories.GetReleaseByTag( - context.Background(), account, pluginName, pluginVersion) - } + // Get the plugin account and repository. + accountRepo := strings.Split(strings.TrimPrefix(splittedURL[0], GitHubURLPrefix), "/") + if len(accountRepo) != NumParts { + cmd.Println( + "Invalid URL. Use the following format: github.com/account/repository@version") + return + } + account = accountRepo[0] + pluginName = accountRepo[1] + if account == "" || pluginName == "" { + cmd.Println( + "Invalid URL. Use the following format: github.com/account/repository@version") + return + } - if err != nil { - cmd.Println("The plugin could not be found: ", err.Error()) - return - } + // Get the release artifact from GitHub. + client = github.NewClient(nil) + var release *github.RepositoryRelease + + if pluginVersion == LatestVersion || pluginVersion == "" { + // Get the latest release. + release, _, err = client.Repositories.GetLatestRelease( + context.Background(), account, pluginName) + } else if strings.HasPrefix(pluginVersion, "v") { + // Get an specific release. + release, _, err = client.Repositories.GetReleaseByTag( + context.Background(), account, pluginName, pluginVersion) + } - if release == nil { - cmd.Println("The plugin could not be found in the release assets") - return - } + if err != nil { + cmd.Println("The plugin could not be found: ", err.Error()) + return + } - // Get the archive extension. - archiveExt := ExtOthers - if runtime.GOOS == "windows" { - archiveExt = ExtWindows - } + if release == nil { + cmd.Println("The plugin could not be found in the release assets") + return + } - // Find and download the plugin binary from the release assets. - pluginFilename, downloadURL, releaseID = findAsset(release, func(name string) bool { - return strings.Contains(name, runtime.GOOS) && - strings.Contains(name, runtime.GOARCH) && - strings.Contains(name, archiveExt) - }) + // Find and download the plugin binary from the release assets. + pluginFilename, downloadURL, releaseID = findAsset(release, func(name string) bool { + return strings.Contains(name, runtime.GOOS) && + strings.Contains(name, runtime.GOARCH) && + strings.Contains(name, string(archiveExt)) + }) + var filePath string + if downloadURL != "" && releaseID != 0 { + cmd.Println("Downloading", downloadURL) + filePath, err = downloadFile(client, account, pluginName, releaseID, pluginFilename) + toBeDeleted = append(toBeDeleted, filePath) + if err != nil { + cmd.Println("Download failed: ", err) + if cleanup { + deleteFiles(toBeDeleted) + } + return + } + cmd.Println("Download completed successfully") + } else { + cmd.Println("The plugin file could not be found in the release assets") + return + } - var filePath string - if downloadURL != "" && releaseID != 0 { - cmd.Println("Downloading", downloadURL) - filePath, err = downloadFile(client, account, pluginName, releaseID, pluginFilename) - toBeDeleted = append(toBeDeleted, filePath) - if err != nil { - cmd.Println("Download failed: ", err) - if cleanup { - deleteFiles(toBeDeleted) + // Find and download the checksums.txt from the release assets. + checksumsFilename, downloadURL, releaseID = findAsset(release, func(name string) bool { + return strings.Contains(name, "checksums.txt") + }) + if checksumsFilename != "" && downloadURL != "" && releaseID != 0 { + cmd.Println("Downloading", downloadURL) + filePath, err = downloadFile(client, account, pluginName, releaseID, checksumsFilename) + toBeDeleted = append(toBeDeleted, filePath) + if err != nil { + cmd.Println("Download failed: ", err) + if cleanup { + deleteFiles(toBeDeleted) + } + return } + cmd.Println("Download completed successfully") + } else { + cmd.Println("The checksum file could not be found in the release assets") return } - cmd.Println("Download completed successfully") - } else { - cmd.Println("The plugin file could not be found in the release assets") - return - } - // Find and download the checksums.txt from the release assets. - checksumsFilename, downloadURL, releaseID = findAsset(release, func(name string) bool { - return strings.Contains(name, "checksums.txt") - }) - if checksumsFilename != "" && downloadURL != "" && releaseID != 0 { - cmd.Println("Downloading", downloadURL) - filePath, err = downloadFile(client, account, pluginName, releaseID, checksumsFilename) - toBeDeleted = append(toBeDeleted, filePath) + // Read the checksums text file. + checksums, err := os.ReadFile(checksumsFilename) if err != nil { - cmd.Println("Download failed: ", err) - if cleanup { - deleteFiles(toBeDeleted) - } + cmd.Println("There was an error reading the checksums file: ", err) return } - cmd.Println("Download completed successfully") - } else { - cmd.Println("The checksum file could not be found in the release assets") - return - } - // Read the checksums text file. - checksums, err := os.ReadFile(checksumsFilename) - if err != nil { - cmd.Println("There was an error reading the checksums file: ", err) - return - } + // Get the checksum for the plugin binary. + sum, err := checksum.SHA256sum(pluginFilename) + if err != nil { + cmd.Println("There was an error calculating the checksum: ", err) + return + } - // Get the checksum for the plugin binary. - sum, err := checksum.SHA256sum(pluginFilename) - if err != nil { - cmd.Println("There was an error calculating the checksum: ", err) - return - } + // Verify the checksums. + checksumLines := strings.Split(string(checksums), "\n") + for _, line := range checksumLines { + if strings.Contains(line, pluginFilename) { + checksum := strings.Split(line, " ")[0] + if checksum != sum { + cmd.Println("Checksum verification failed") + return + } - // Verify the checksums. - checksumLines := strings.Split(string(checksums), "\n") - for _, line := range checksumLines { - if strings.Contains(line, pluginFilename) { - checksum := strings.Split(line, " ")[0] - if checksum != sum { - cmd.Println("Checksum verification failed") - return + cmd.Println("Checksum verification passed") + break } - - cmd.Println("Checksum verification passed") - break } - } - if pullOnly { - cmd.Println("Plugin binary downloaded to", pluginFilename) - // Only the checksums file will be deleted if the --pull-only flag is set. - if err := os.Remove(checksumsFilename); err != nil { - cmd.Println("There was an error deleting the file: ", err) + if pullOnly { + cmd.Println("Plugin binary downloaded to", pluginFilename) + // Only the checksums file will be deleted if the --pull-only flag is set. + if err := os.Remove(checksumsFilename); err != nil { + cmd.Println("There was an error deleting the file: ", err) + } + return } - return + default: + cmd.Println("Invalid URL or file path") } - // Create a new gatewayd_plugins.yaml file if it doesn't exist. + // NOTE: The rest of the code is executed regardless of the source, + // since the plugin binary is already available (or downloaded) at this point. + + // Create a new "gatewayd_plugins.yaml" file if it doesn't exist. if _, err := os.Stat(pluginConfigFile); os.IsNotExist(err) { generateConfig(cmd, Plugins, pluginConfigFile, false) } else { @@ -649,22 +689,22 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { } } - // Read the gatewayd_plugins.yaml file. + // Read the "gatewayd_plugins.yaml" file. pluginsConfig, err := os.ReadFile(pluginConfigFile) if err != nil { - log.Println(err) + cmd.Println(err) return } // Get the registered plugins from the plugins configuration file. var localPluginsConfig map[string]interface{} if err := yamlv3.Unmarshal(pluginsConfig, &localPluginsConfig); err != nil { - log.Println("Failed to unmarshal the plugins configuration file: ", err) + cmd.Println("Failed to unmarshal the plugins configuration file: ", err) return } pluginsList, ok := localPluginsConfig["plugins"].([]interface{}) //nolint:varnamelen if !ok { - log.Println("There was an error reading the plugins file from disk") + cmd.Println("There was an error reading the plugins file from disk") return } @@ -710,10 +750,14 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { // Extract the archive. var filenames []string - if runtime.GOOS == "windows" { + switch archiveExt { + case ExtensionZip: filenames, err = extractZip(pluginFilename, pluginOutputDir) - } else { + case ExtensionTarGz: filenames, err = extractTarGz(pluginFilename, pluginOutputDir) + default: + cmd.Println("Invalid archive extension") + return } if err != nil { @@ -753,7 +797,7 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { } var contents string - if strings.HasPrefix(pluginURL, GitHubURLPrefix) { + if source == SourceGitHub { // Get the list of files in the repository. var repoContents *github.RepositoryContent repoContents, _, _, err = client.Repositories.GetContents( diff --git a/gatewayd_plugins.yaml b/gatewayd_plugins.yaml index 5beaa108..02e6d05b 100644 --- a/gatewayd_plugins.yaml +++ b/gatewayd_plugins.yaml @@ -79,7 +79,7 @@ startTimeout: 1m plugins: - name: gatewayd-plugin-cache enabled: True - url: github.com/gatewayd/gatewayd-plugin-cache@latest + url: github.com/gatewayd-io/gatewayd-plugin-cache@latest localPath: ../gatewayd-plugin-cache/gatewayd-plugin-cache args: ["--log-level", "debug"] env: From cac99a66f2a41bb2842698e9d126ea8276f4a8d7 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Wed, 24 Jan 2024 22:33:48 +0100 Subject: [PATCH 05/25] Fix linter errors --- cmd/utils.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/cmd/utils.go b/cmd/utils.go index 760a4b61..0d1ec4ea 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -442,7 +442,7 @@ func downloadFile( func deleteFiles(toBeDeleted []string) { for _, filename := range toBeDeleted { if err := os.Remove(filename); err != nil { - fmt.Println("There was an error deleting the file: ", err) + fmt.Println("There was an error deleting the file: ", err) //nolint:forbidigo return } } @@ -484,13 +484,13 @@ func getFileExtension() Extension { func installPlugin(cmd *cobra.Command, pluginURL string) { var ( // This is a list of files that will be deleted after the plugin is installed. - toBeDeleted []string = []string{} + toBeDeleted = []string{} // Source of the plugin: file or GitHub. - source Source = detectSource(pluginURL) + source = detectSource(pluginURL) // The extension of the archive based on the OS: .zip or .tar.gz. - archiveExt Extension = getFileExtension() + archiveExt = getFileExtension() releaseID int64 downloadURL string @@ -666,6 +666,7 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { } return } + case SourceUnknown: default: cmd.Println("Invalid URL or file path") } @@ -676,16 +677,14 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { // Create a new "gatewayd_plugins.yaml" file if it doesn't exist. if _, err := os.Stat(pluginConfigFile); os.IsNotExist(err) { generateConfig(cmd, Plugins, pluginConfigFile, false) - } else { + } else if !backupConfig && !noPrompt { // If the config file exists, we should prompt the user to backup // the plugins configuration file. - if !backupConfig && !noPrompt { - cmd.Print("Do you want to backup the plugins configuration file? [Y/n] ") - var backupOption string - _, err := fmt.Scanln(&backupOption) - if err == nil && (backupOption == "y" || backupOption == "Y") { - backupConfig = true - } + cmd.Print("Do you want to backup the plugins configuration file? [Y/n] ") + var backupOption string + _, err := fmt.Scanln(&backupOption) + if err == nil && (backupOption == "y" || backupOption == "Y") { + backupConfig = true } } From 4348cf7ea12627af4aed71eb3372366af0f0ffd2 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Wed, 24 Jan 2024 22:57:01 +0100 Subject: [PATCH 06/25] Use cast to refactor casts --- cmd/utils.go | 63 +++++++++++++++++++++------------------------------- 1 file changed, 25 insertions(+), 38 deletions(-) diff --git a/cmd/utils.go b/cmd/utils.go index 0d1ec4ea..50b40d41 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -28,6 +28,7 @@ import ( koanfJson "github.com/knadh/koanf/parsers/json" "github.com/knadh/koanf/parsers/yaml" jsonSchemaV5 "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/spf13/cast" "github.com/spf13/cobra" yamlv3 "gopkg.in/yaml.v3" ) @@ -701,11 +702,7 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { cmd.Println("Failed to unmarshal the plugins configuration file: ", err) return } - pluginsList, ok := localPluginsConfig["plugins"].([]interface{}) //nolint:varnamelen - if !ok { - cmd.Println("There was an error reading the plugins file from disk") - return - } + pluginsList := cast.ToSlice(localPluginsConfig["plugins"]) // Check if the plugin is already installed. for _, plugin := range pluginsList { @@ -714,27 +711,25 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { break } - //nolint:nestif - if pluginInstance, ok := plugin.(map[string]interface{}); ok { - if pluginInstance["name"] == pluginName { - // Show a list of options to the user. - cmd.Println("Plugin is already installed.") - if !noPrompt { - cmd.Print("Do you want to update the plugin? [y/N] ") + pluginInstance := cast.ToStringMap(plugin) + if pluginInstance["name"] == pluginName { + // Show a list of options to the user. + cmd.Println("Plugin is already installed.") + if !noPrompt { + cmd.Print("Do you want to update the plugin? [y/N] ") - var updateOption string - _, err := fmt.Scanln(&updateOption) - if err == nil && (updateOption == "y" || updateOption == "Y") { - break - } + var updateOption string + _, err := fmt.Scanln(&updateOption) + if err == nil && (updateOption == "y" || updateOption == "Y") { + break } + } - cmd.Println("Aborting...") - if cleanup { - deleteFiles(toBeDeleted) - } - return + cmd.Println("Aborting...") + if cleanup { + deleteFiles(toBeDeleted) } + return } } @@ -831,17 +826,10 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { cmd.Println("Failed to unmarshal the downloaded plugins configuration file: ", err) return } - defaultPluginConfig, ok := downloadedPluginConfig["plugins"].([]interface{}) - if !ok { - cmd.Println("There was an error reading the plugins file from the repository") - return - } + defaultPluginConfig := cast.ToSlice(downloadedPluginConfig["plugins"]) + // Get the plugin configuration. - pluginConfig, ok := defaultPluginConfig[0].(map[string]interface{}) - if !ok { - cmd.Println("There was an error reading the default plugin configuration") - return - } + pluginConfig := cast.ToStringMap(defaultPluginConfig[0]) // Update the plugin's local path and checksum. pluginConfig["localPath"] = localPath @@ -850,12 +838,11 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { // Add the plugin config to the list of plugin configs. added := false for idx, plugin := range pluginsList { - if pluginInstance, ok := plugin.(map[string]interface{}); ok { - if pluginInstance["name"] == pluginName { - pluginsList[idx] = pluginConfig - added = true - break - } + pluginInstance := cast.ToStringMap(plugin) + if pluginInstance["name"] == pluginName { + pluginsList[idx] = pluginConfig + added = true + break } } if !added { From aeb6d85d8525529456f43c04fbfd702ec807bedb Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Wed, 24 Jan 2024 23:09:44 +0100 Subject: [PATCH 07/25] Test using the latest version of the plugin --- cmd/run_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/run_test.go b/cmd/run_test.go index ebaae822..9672f37c 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -199,11 +199,13 @@ func Test_runCmdWithCachePlugin(t *testing.T) { // Test plugin install command. output, err := executeCommandC( rootCmd, "plugin", "install", - "github.com/gatewayd-io/gatewayd-plugin-cache@v0.2.4", + "github.com/gatewayd-io/gatewayd-plugin-cache@latest", "-p", pluginTestConfigFile, "--update") require.NoError(t, err, "plugin install should not return an error") - assert.Contains(t, output, "Downloading https://github.com/gatewayd-io/gatewayd-plugin-cache/releases/download/v0.2.4/gatewayd-plugin-cache-linux-amd64-v0.2.4.tar.gz") //nolint:lll - assert.Contains(t, output, "Downloading https://github.com/gatewayd-io/gatewayd-plugin-cache/releases/download/v0.2.4/checksums.txt") //nolint:lll + assert.Contains(t, output, "Installing plugin from CLI argument") + assert.Contains(t, output, "Downloading ") + assert.Contains(t, output, "gatewayd-plugin-cache-linux-amd64-") + assert.Contains(t, output, "/checksums.txt") assert.Contains(t, output, "Download completed successfully") assert.Contains(t, output, "Checksum verification passed") assert.Contains(t, output, "Plugin binary extracted to plugins/gatewayd-plugin-cache") From 28e045646c669860faa68062bbe97786f7e14686 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Thu, 25 Jan 2024 19:03:42 +0100 Subject: [PATCH 08/25] Check if the plugins are installed before updating or installing them Fix user input scanner (use a slightly better syntax) Add maps to allow list of depguard --- .golangci.yaml | 1 + cmd/plugin_install.go | 69 +++++++++++++++++++++++++++++++++++++++---- cmd/utils.go | 6 ++-- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index bebf047b..9ef208e2 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -61,6 +61,7 @@ linters-settings: - "github.com/google/go-cmp" - "github.com/google/go-github/v53/github" - "github.com/codingsince1985/checksum" + - "golang.org/x/exp/maps" test: files: - $test diff --git a/cmd/plugin_install.go b/cmd/plugin_install.go index 61ddcab6..daeeef74 100644 --- a/cmd/plugin_install.go +++ b/cmd/plugin_install.go @@ -1,12 +1,15 @@ package cmd import ( + "fmt" "os" + "strings" "github.com/gatewayd-io/gatewayd/config" "github.com/getsentry/sentry-go" "github.com/spf13/cast" "github.com/spf13/cobra" + "golang.org/x/exp/maps" yamlv3 "gopkg.in/yaml.v3" ) @@ -89,7 +92,8 @@ var pluginInstallCmd = &cobra.Command{ pluginsList := cast.ToSlice(localPluginsConfig["plugins"]) // Get the list of plugin download URLs. - var pluginURLs []string + pluginURLs := map[string]string{} + existingPluginURLs := map[string]string{} for _, plugin := range pluginsList { // Get the plugin instance. pluginInstance := cast.ToStringMapString(plugin) @@ -97,18 +101,71 @@ var pluginInstallCmd = &cobra.Command{ // Append the plugin URL to the list of plugin URLs. name := cast.ToString(pluginInstance["name"]) url := cast.ToString(pluginInstance["url"]) - if url != "" { - pluginURLs = append(pluginURLs, url) - } else { + if url == "" { cmd.Println("Plugin URL or file path not found in the plugins configuration file for", name) return } + + // Check if duplicate plugin names exist in the plugins configuration file. + if _, ok := pluginURLs[name]; ok { + cmd.Println("Duplicate plugin name found in the plugins configuration file:", name) + return + } + + // Update list of plugin URLs based on + // whether the plugin is already installed or not. + localPath := cast.ToString(pluginInstance["localPath"]) + if _, err := os.Stat(localPath); err == nil { + existingPluginURLs[name] = url + } else { + pluginURLs[name] = url + } + } + + // Check if the plugin is already installed and prompt the user to confirm the update. + if len(existingPluginURLs) > 0 { + pluginNames := strings.Join(maps.Keys[map[string]string](existingPluginURLs), ", ") + cmd.Printf("The following plugins are already installed: %s\n", pluginNames) + + if noPrompt { + if !update { + cmd.Println("Use the --update flag to update the plugins") + cmd.Println("Aborting...") + return + } else { + // Merge the existing plugin URLs with the plugin URLs. + for name, url := range existingPluginURLs { + pluginURLs[name] = url + } + } + } else { + cmd.Print("Do you want to update the existing plugins? [y/N] ") + var response string + _, err := fmt.Scanln(&response) + if err == nil && strings.ToLower(response) == "y" { + // Set the update flag to true, so that the installPlugin function + // can update the existing plugins and doesn't ask for user input again. + update = true + + // Merge the existing plugin URLs with the plugin URLs. + for name, url := range existingPluginURLs { + pluginURLs[name] = url + } + } else { + cmd.Println("Existing plugins will not be updated") + } + } } // Validate the plugin URLs. if len(args) == 0 && len(pluginURLs) == 0 { - cmd.Println( - "No plugin URLs or file path found in the plugins configuration file or CLI argument") + if len(existingPluginURLs) > 0 && !update { + cmd.Println("Use the --update flag to update the plugins") + } else { + cmd.Println( + "No plugin URLs or file path found in the plugins configuration file or CLI argument") + cmd.Println("Aborting...") + } return } diff --git a/cmd/utils.go b/cmd/utils.go index 50b40d41..965dae4c 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -684,7 +684,9 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { cmd.Print("Do you want to backup the plugins configuration file? [Y/n] ") var backupOption string _, err := fmt.Scanln(&backupOption) - if err == nil && (backupOption == "y" || backupOption == "Y") { + if err == nil && strings.ToLower(backupOption) == "n" { + backupConfig = false + } else { backupConfig = true } } @@ -720,7 +722,7 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { var updateOption string _, err := fmt.Scanln(&updateOption) - if err == nil && (updateOption == "y" || updateOption == "Y") { + if err != nil && strings.ToLower(updateOption) == "y" { break } } From 9016f3642d83821c0591fe986b3f03429fb5941f Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Thu, 25 Jan 2024 19:16:58 +0100 Subject: [PATCH 09/25] Fix linter errors --- .golangci.yaml | 6 ++++++ cmd/plugin_install.go | 10 +++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 9ef208e2..1437f382 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -43,6 +43,8 @@ linters-settings: - "github.com/getsentry/sentry-go" - "github.com/rs/zerolog" - "github.com/grpc-ecosystem/grpc-gateway" + - "google.golang.org/grpc" + - "google.golang.org/protobuf" - "github.com/knadh/koanf" - "github.com/panjf2000/gnet/v2" - "github.com/spf13/cobra" @@ -62,6 +64,10 @@ linters-settings: - "github.com/google/go-github/v53/github" - "github.com/codingsince1985/checksum" - "golang.org/x/exp/maps" + - "golang.org/x/exp/slices" + - "gopkg.in/yaml.v3" + - "github.com/zenizh/go-capturer" + - "gopkg.in/natefinch/lumberjack.v2" test: files: - $test diff --git a/cmd/plugin_install.go b/cmd/plugin_install.go index daeeef74..bb9ac988 100644 --- a/cmd/plugin_install.go +++ b/cmd/plugin_install.go @@ -132,11 +132,11 @@ var pluginInstallCmd = &cobra.Command{ cmd.Println("Use the --update flag to update the plugins") cmd.Println("Aborting...") return - } else { - // Merge the existing plugin URLs with the plugin URLs. - for name, url := range existingPluginURLs { - pluginURLs[name] = url - } + } + + // Merge the existing plugin URLs with the plugin URLs. + for name, url := range existingPluginURLs { + pluginURLs[name] = url } } else { cmd.Print("Do you want to update the existing plugins? [y/N] ") From d14180469fe2680f9bd366e0f67407831fed8993 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Thu, 25 Jan 2024 21:37:59 +0100 Subject: [PATCH 10/25] Add overwrite-config flag to prevent updating config for automatic installation --- cmd/plugin_install.go | 3 +++ cmd/utils.go | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/plugin_install.go b/cmd/plugin_install.go index bb9ac988..c65f5b79 100644 --- a/cmd/plugin_install.go +++ b/cmd/plugin_install.go @@ -43,6 +43,7 @@ var ( backupConfig bool noPrompt bool pluginName string + overwriteConfig bool ) // pluginInstallCmd represents the plugin install command. @@ -202,6 +203,8 @@ func init() { &backupConfig, "backup", false, "Backup the plugins configuration file before installing the plugin") pluginInstallCmd.Flags().StringVarP( &pluginName, "name", "n", "", "Name of the plugin") + pluginInstallCmd.Flags().BoolVar( + &overwriteConfig, "overwrite-config", true, "Overwrite the existing plugins configuration file") pluginInstallCmd.Flags().BoolVar( &enableSentry, "sentry", true, "Enable Sentry") // Already exists in run.go } diff --git a/cmd/utils.go b/cmd/utils.go index 965dae4c..a7393778 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -861,10 +861,12 @@ func installPlugin(cmd *cobra.Command, pluginURL string) { return } - // Write the YAML to the plugins config file. - if err = os.WriteFile(pluginConfigFile, updatedPlugins, FilePermissions); err != nil { - cmd.Println("There was an error writing the plugins configuration file: ", err) - return + // Write the YAML to the plugins config file if the --overwrite-config flag is set. + if overwriteConfig { + if err = os.WriteFile(pluginConfigFile, updatedPlugins, FilePermissions); err != nil { + cmd.Println("There was an error writing the plugins configuration file: ", err) + return + } } // Delete the downloaded and extracted files, except the plugin binary, From 9b311cebce0b7fd89619fc54468e1a37632b476e Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 26 Jan 2024 20:19:10 +0100 Subject: [PATCH 11/25] Fix help for the plugin install command --- cmd/plugin_install.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/plugin_install.go b/cmd/plugin_install.go index c65f5b79..5e01e9f2 100644 --- a/cmd/plugin_install.go +++ b/cmd/plugin_install.go @@ -50,7 +50,7 @@ var ( var pluginInstallCmd = &cobra.Command{ Use: "install", Short: "Install a plugin from a local archive or a GitHub repository", - Example: " gatewayd plugin install github.com/gatewayd-io/gatewayd-plugin-cache@latest", + Example: " gatewayd plugin install ", //nolint:lll Run: func(cmd *cobra.Command, args []string) { // Enable Sentry. if enableSentry { From 45a6ce684c1c5f5bdd917622e269e6ac9587d84f Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 26 Jan 2024 20:42:32 +0100 Subject: [PATCH 12/25] Fix variable reuse and naming --- cmd/utils.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/utils.go b/cmd/utils.go index a7393778..fdac79d1 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -218,22 +218,21 @@ func extractZip(filename, dest string) ([]string, error) { // Extract the files. filenames := []string{} - for _, file := range zipRc.File { - switch fileInfo := file.FileInfo(); { + for _, fileOrDir := range zipRc.File { + switch fileInfo := fileOrDir.FileInfo(); { case fileInfo.IsDir(): // Sanitize the path. - filename := filepath.Clean(file.Name) - if !path.IsAbs(filename) { - destPath := path.Join(dest, filename) + dirName := filepath.Clean(fileOrDir.Name) + if !path.IsAbs(dirName) { // Create the directory. - + destPath := path.Join(dest, dirName) if err := os.MkdirAll(destPath, FolderPermissions); err != nil { return nil, gerr.ErrExtractFailed.Wrap(err) } } case fileInfo.Mode().IsRegular(): // Sanitize the path. - outFilename := filepath.Join(filepath.Clean(dest), filepath.Clean(file.Name)) + outFilename := filepath.Join(filepath.Clean(dest), filepath.Clean(fileOrDir.Name)) // Check for ZipSlip. if strings.HasPrefix(outFilename, string(os.PathSeparator)) { @@ -249,7 +248,7 @@ func extractZip(filename, dest string) ([]string, error) { defer outFile.Close() // Open the file in the zip archive. - fileRc, err := file.Open() + fileRc, err := fileOrDir.Open() if err != nil { os.Remove(outFilename) return nil, gerr.ErrExtractFailed.Wrap(err) @@ -261,7 +260,7 @@ func extractZip(filename, dest string) ([]string, error) { return nil, gerr.ErrExtractFailed.Wrap(err) } - fileMode := file.FileInfo().Mode() + fileMode := fileOrDir.FileInfo().Mode() // Set the file permissions. if fileMode.IsRegular() && fileMode&ExecFileMask != 0 { if err := os.Chmod(outFilename, ExecFilePermissions); err != nil { @@ -276,7 +275,7 @@ func extractZip(filename, dest string) ([]string, error) { filenames = append(filenames, outFile.Name()) default: return nil, gerr.ErrExtractFailed.Wrap( - fmt.Errorf("unknown file type: %s", file.Name)) + fmt.Errorf("unknown file type: %s", fileOrDir.Name)) } } From 25614039b2d8d4585e3dbd223212745ca996ce8b Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 26 Jan 2024 21:26:33 +0100 Subject: [PATCH 13/25] Add test for automated installation with no overwrite --- cmd/plugin_install_test.go | 32 ++++++++++++++++++++++++++++++ cmd/testdata/gatewayd_plugins.yaml | 21 ++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 cmd/testdata/gatewayd_plugins.yaml diff --git a/cmd/plugin_install_test.go b/cmd/plugin_install_test.go index 5c960fc7..78525468 100644 --- a/cmd/plugin_install_test.go +++ b/cmd/plugin_install_test.go @@ -51,3 +51,35 @@ func Test_pluginInstallCmd(t *testing.T) { require.NoError(t, os.Remove(pluginTestConfigFile)) require.NoError(t, os.Remove(fmt.Sprintf("%s.bak", pluginTestConfigFile))) } + +func Test_pluginInstallCmdAutomatedNoOverwrite(t *testing.T) { + pluginTestConfig := "testdata/gatewayd_plugins.yaml" + + // Test plugin install command. + output, err := executeCommandC( + rootCmd, "plugin", "install", + "-p", pluginTestConfig, "--update", "--backup", "--overwrite-config=false") + require.NoError(t, err, "plugin install should not return an error") + assert.Contains(t, output, "/gatewayd-plugin-cache-linux-amd64-") + assert.Contains(t, output, "/checksums.txt") + assert.Contains(t, output, "Download completed successfully") + assert.Contains(t, output, "Checksum verification passed") + assert.Contains(t, output, "Plugin binary extracted to plugins/gatewayd-plugin-cache") + assert.Contains(t, output, "Plugin installed successfully") + + // See if the plugin was actually installed. + output, err = executeCommandC(rootCmd, "plugin", "list", "-p", pluginTestConfig) + require.NoError(t, err, "plugin list should not return an error") + assert.Contains(t, output, "Name: gatewayd-plugin-cache") + + // Clean up. + assert.FileExists(t, "plugins/gatewayd-plugin-cache") + assert.FileExists(t, fmt.Sprintf("%s.bak", pluginTestConfig)) + assert.NoFileExists(t, "plugins/LICENSE") + assert.NoFileExists(t, "plugins/README.md") + assert.NoFileExists(t, "plugins/checksum.txt") + assert.NoFileExists(t, "plugins/gatewayd_plugin.yaml") + + require.NoError(t, os.RemoveAll("plugins/")) + require.NoError(t, os.Remove(fmt.Sprintf("%s.bak", pluginTestConfig))) +} diff --git a/cmd/testdata/gatewayd_plugins.yaml b/cmd/testdata/gatewayd_plugins.yaml new file mode 100644 index 00000000..3e3e0dec --- /dev/null +++ b/cmd/testdata/gatewayd_plugins.yaml @@ -0,0 +1,21 @@ +plugins: + - name: gatewayd-plugin-cache + enabled: True + url: github.com/gatewayd-io/gatewayd-plugin-cache@v0.2.10 + localPath: plugins/gatewayd-plugin-cache + args: ["--log-level", "debug"] + env: + - MAGIC_COOKIE_KEY=GATEWAYD_PLUGIN + - MAGIC_COOKIE_VALUE=5712b87aa5d7e9f9e9ab643e6603181c5b796015cb1c09d6f5ada882bf2a1872 + - REDIS_URL=redis://localhost:6379/0 + - EXPIRY=1h + - METRICS_ENABLED=True + - METRICS_UNIX_DOMAIN_SOCKET=/tmp/gatewayd-plugin-cache.sock + - METRICS_PATH=/metrics + - PERIODIC_INVALIDATOR_ENABLED=True + - PERIODIC_INVALIDATOR_INTERVAL=1m + - PERIODIC_INVALIDATOR_START_DELAY=1m + - API_ADDRESS=localhost:18080 + - EXIT_ON_STARTUP_ERROR=False + - SENTRY_DSN=https://70eb1abcd32e41acbdfc17bc3407a543@o4504550475038720.ingest.sentry.io/4505342961123328 + checksum: 867e09326da10b6e321d8dbcfcf2d20835bde79a82edc2b440dd81d151041672 From 38d9ecdd2a66408af80a7d4ebae9633028b52342 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sat, 27 Jan 2024 12:59:10 +0100 Subject: [PATCH 14/25] Generate and use separate plugins config file for each test --- cmd/cmd_helpers_test.go | 6 ------ cmd/config_init_test.go | 1 + cmd/config_lint_test.go | 1 + cmd/plugin_init_test.go | 1 + cmd/plugin_install_test.go | 12 +++++++----- cmd/plugin_list_test.go | 1 + cmd/run_test.go | 20 +++++++++++++++----- 7 files changed, 26 insertions(+), 16 deletions(-) diff --git a/cmd/cmd_helpers_test.go b/cmd/cmd_helpers_test.go index 8e5657c2..e8ed00bc 100644 --- a/cmd/cmd_helpers_test.go +++ b/cmd/cmd_helpers_test.go @@ -6,12 +6,6 @@ import ( "github.com/spf13/cobra" ) -var ( - globalTestConfigFile = "./test_global.yaml" - globalTLSTestConfigFile = "./testdata/gatewayd_tls.yaml" - pluginTestConfigFile = "./test_plugins.yaml" -) - // executeCommandC executes a cobra command and returns the command, output, and error. // Taken from https://github.com/spf13/cobra/blob/0c72800b8dba637092b57a955ecee75949e79a73/command_test.go#L48. func executeCommandC(root *cobra.Command, args ...string) (string, error) { diff --git a/cmd/config_init_test.go b/cmd/config_init_test.go index 351acf07..456db54f 100644 --- a/cmd/config_init_test.go +++ b/cmd/config_init_test.go @@ -10,6 +10,7 @@ import ( ) func Test_configInitCmd(t *testing.T) { + globalTestConfigFile := "./test_global_configInitCmd.yaml" // Test configInitCmd. output, err := executeCommandC(rootCmd, "config", "init", "-c", globalTestConfigFile) require.NoError(t, err, "configInitCmd should not return an error") diff --git a/cmd/config_lint_test.go b/cmd/config_lint_test.go index 32e2a7ab..19030df7 100644 --- a/cmd/config_lint_test.go +++ b/cmd/config_lint_test.go @@ -10,6 +10,7 @@ import ( ) func Test_configLintCmd(t *testing.T) { + globalTestConfigFile := "./test_global_configLintCmd.yaml" // Test configInitCmd. output, err := executeCommandC(rootCmd, "config", "init", "-c", globalTestConfigFile) require.NoError(t, err, "configInitCmd should not return an error") diff --git a/cmd/plugin_init_test.go b/cmd/plugin_init_test.go index 5adef60a..2d9abb32 100644 --- a/cmd/plugin_init_test.go +++ b/cmd/plugin_init_test.go @@ -10,6 +10,7 @@ import ( ) func Test_pluginInitCmd(t *testing.T) { + pluginTestConfigFile := "./test_plugins_pluginInitCmd.yaml" // Test plugin init command. output, err := executeCommandC(rootCmd, "plugin", "init", "-p", pluginTestConfigFile) require.NoError(t, err, "plugin init command should not have returned an error") diff --git a/cmd/plugin_install_test.go b/cmd/plugin_install_test.go index 78525468..1af6dcba 100644 --- a/cmd/plugin_install_test.go +++ b/cmd/plugin_install_test.go @@ -10,6 +10,8 @@ import ( ) func Test_pluginInstallCmd(t *testing.T) { + pluginTestConfigFile := "./test_plugins_pluginInstallCmd.yaml" + // Create a test plugin config file. output, err := executeCommandC(rootCmd, "plugin", "init", "-p", pluginTestConfigFile) require.NoError(t, err, "plugin init should not return an error") @@ -53,12 +55,12 @@ func Test_pluginInstallCmd(t *testing.T) { } func Test_pluginInstallCmdAutomatedNoOverwrite(t *testing.T) { - pluginTestConfig := "testdata/gatewayd_plugins.yaml" + pluginTestConfigFile := "./testdata/gatewayd_plugins.yaml" // Test plugin install command. output, err := executeCommandC( rootCmd, "plugin", "install", - "-p", pluginTestConfig, "--update", "--backup", "--overwrite-config=false") + "-p", pluginTestConfigFile, "--update", "--backup", "--overwrite-config=false") require.NoError(t, err, "plugin install should not return an error") assert.Contains(t, output, "/gatewayd-plugin-cache-linux-amd64-") assert.Contains(t, output, "/checksums.txt") @@ -68,18 +70,18 @@ func Test_pluginInstallCmdAutomatedNoOverwrite(t *testing.T) { assert.Contains(t, output, "Plugin installed successfully") // See if the plugin was actually installed. - output, err = executeCommandC(rootCmd, "plugin", "list", "-p", pluginTestConfig) + output, err = executeCommandC(rootCmd, "plugin", "list", "-p", pluginTestConfigFile) require.NoError(t, err, "plugin list should not return an error") assert.Contains(t, output, "Name: gatewayd-plugin-cache") // Clean up. assert.FileExists(t, "plugins/gatewayd-plugin-cache") - assert.FileExists(t, fmt.Sprintf("%s.bak", pluginTestConfig)) + assert.FileExists(t, fmt.Sprintf("%s.bak", pluginTestConfigFile)) assert.NoFileExists(t, "plugins/LICENSE") assert.NoFileExists(t, "plugins/README.md") assert.NoFileExists(t, "plugins/checksum.txt") assert.NoFileExists(t, "plugins/gatewayd_plugin.yaml") require.NoError(t, os.RemoveAll("plugins/")) - require.NoError(t, os.Remove(fmt.Sprintf("%s.bak", pluginTestConfig))) + require.NoError(t, os.Remove(fmt.Sprintf("%s.bak", pluginTestConfigFile))) } diff --git a/cmd/plugin_list_test.go b/cmd/plugin_list_test.go index 7615d9ff..6298052d 100644 --- a/cmd/plugin_list_test.go +++ b/cmd/plugin_list_test.go @@ -10,6 +10,7 @@ import ( ) func Test_pluginListCmd(t *testing.T) { + pluginTestConfigFile := "./test_plugins_pluginListCmd.yaml" // Test plugin list command. output, err := executeCommandC(rootCmd, "plugin", "init", "-p", pluginTestConfigFile) require.NoError(t, err, "plugin init command should not have returned an error") diff --git a/cmd/run_test.go b/cmd/run_test.go index 9672f37c..3d877cb9 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -13,7 +13,11 @@ import ( "github.com/zenizh/go-capturer" ) +var waitBeforeStop = time.Second + func Test_runCmd(t *testing.T) { + globalTestConfigFile := "./test_global_runCmd.yaml" + pluginTestConfigFile := "./test_plugins_runCmd.yaml" // Create a test plugins config file. _, err := executeCommandC(rootCmd, "plugin", "init", "--force", "-p", pluginTestConfigFile) require.NoError(t, err, "plugin init command should not have returned an error") @@ -45,7 +49,7 @@ func Test_runCmd(t *testing.T) { waitGroup.Add(1) go func(waitGroup *sync.WaitGroup) { - time.Sleep(100 * time.Millisecond) + time.Sleep(waitBeforeStop) StopGracefully( context.Background(), @@ -70,6 +74,8 @@ func Test_runCmd(t *testing.T) { // Test_runCmdWithTLS tests the run command with TLS enabled on the server. func Test_runCmdWithTLS(t *testing.T) { + globalTLSTestConfigFile := "./testdata/gatewayd_tls.yaml" + pluginTestConfigFile := "./test_plugins_runCmdWithTLS.yaml" // Create a test plugins config file. _, err := executeCommandC(rootCmd, "plugin", "init", "--force", "-p", pluginTestConfigFile) require.NoError(t, err, "plugin init command should not have returned an error") @@ -101,7 +107,7 @@ func Test_runCmdWithTLS(t *testing.T) { waitGroup.Add(1) go func(waitGroup *sync.WaitGroup) { - time.Sleep(100 * time.Millisecond) + time.Sleep(waitBeforeStop) StopGracefully( context.Background(), @@ -126,6 +132,8 @@ func Test_runCmdWithTLS(t *testing.T) { // Test_runCmdWithMultiTenancy tests the run command with multi-tenancy enabled. // Note: This test needs two instances of PostgreSQL running on ports 5432 and 5433. func Test_runCmdWithMultiTenancy(t *testing.T) { + globalTestConfigFile := "testdata/gatewayd.yaml" + pluginTestConfigFile := "./test_plugins_runCmdWithMultiTenancy.yaml" // Create a test plugins config file. _, err := executeCommandC(rootCmd, "plugin", "init", "--force", "-p", pluginTestConfigFile) require.NoError(t, err, "plugin init command should not have returned an error") @@ -140,7 +148,7 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { // Test run command. output := capturer.CaptureOutput(func() { _, err := executeCommandC( - rootCmd, "run", "-c", "testdata/gatewayd.yaml", "-p", pluginTestConfigFile) + rootCmd, "run", "-c", globalTestConfigFile, "-p", pluginTestConfigFile) require.NoError(t, err, "run command should not have returned an error") }) // Print the output for debugging purposes. @@ -158,7 +166,7 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { waitGroup.Add(1) go func(waitGroup *sync.WaitGroup) { - time.Sleep(500 * time.Millisecond) + time.Sleep(waitBeforeStop) StopGracefully( context.Background(), @@ -181,6 +189,8 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { } func Test_runCmdWithCachePlugin(t *testing.T) { + globalTestConfigFile := "./test_global_runCmdWithCachePlugin.yaml" + pluginTestConfigFile := "./test_plugins_runCmdWithCachePlugin.yaml" // TODO: Remove this once these global variables are removed from cmd/run.go. // https://github.com/gatewayd-io/gatewayd/issues/324 stopChan = make(chan struct{}) @@ -236,7 +246,7 @@ func Test_runCmdWithCachePlugin(t *testing.T) { waitGroup.Add(1) go func(waitGroup *sync.WaitGroup) { - time.Sleep(time.Second) + time.Sleep(waitBeforeStop) StopGracefully( context.Background(), From 42d6a9a976d3749510db3bb5038aaeca87226ae5 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sat, 27 Jan 2024 13:04:26 +0100 Subject: [PATCH 15/25] Don't remove config files when running multiple goroutines --- cmd/run_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/cmd/run_test.go b/cmd/run_test.go index 3d877cb9..e03519db 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "os" "sync" "testing" "time" @@ -66,10 +65,6 @@ func Test_runCmd(t *testing.T) { }(&waitGroup) waitGroup.Wait() - - // Clean up. - require.NoError(t, os.Remove(pluginTestConfigFile)) - require.NoError(t, os.Remove(globalTestConfigFile)) } // Test_runCmdWithTLS tests the run command with TLS enabled on the server. @@ -124,9 +119,6 @@ func Test_runCmdWithTLS(t *testing.T) { }(&waitGroup) waitGroup.Wait() - - // Clean up. - require.NoError(t, os.Remove(pluginTestConfigFile)) } // Test_runCmdWithMultiTenancy tests the run command with multi-tenancy enabled. @@ -183,9 +175,6 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { }(&waitGroup) waitGroup.Wait() - - // Clean up. - require.NoError(t, os.Remove(pluginTestConfigFile)) } func Test_runCmdWithCachePlugin(t *testing.T) { @@ -263,9 +252,4 @@ func Test_runCmdWithCachePlugin(t *testing.T) { }(&waitGroup) waitGroup.Wait() - - // Clean up. - require.NoError(t, os.RemoveAll("plugins/")) - require.NoError(t, os.Remove(pluginTestConfigFile)) - require.NoError(t, os.Remove(globalTestConfigFile)) } From b47923a129e2bd19366c7f5bb8a824b7bc3aa199 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sat, 27 Jan 2024 20:30:49 +0100 Subject: [PATCH 16/25] Try to fix the race issue --- cmd/run_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/run_test.go b/cmd/run_test.go index e03519db..f0b6a481 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -198,8 +198,8 @@ func Test_runCmdWithCachePlugin(t *testing.T) { // Test plugin install command. output, err := executeCommandC( rootCmd, "plugin", "install", - "github.com/gatewayd-io/gatewayd-plugin-cache@latest", - "-p", pluginTestConfigFile, "--update") + "-p", pluginTestConfigFile, "--update", + "github.com/gatewayd-io/gatewayd-plugin-cache@latest") require.NoError(t, err, "plugin install should not return an error") assert.Contains(t, output, "Installing plugin from CLI argument") assert.Contains(t, output, "Downloading ") @@ -235,7 +235,7 @@ func Test_runCmdWithCachePlugin(t *testing.T) { waitGroup.Add(1) go func(waitGroup *sync.WaitGroup) { - time.Sleep(waitBeforeStop) + time.Sleep(waitBeforeStop * 2) StopGracefully( context.Background(), From 60f810f88c742c64c7896e4bbf4be7ca619755b0 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sat, 27 Jan 2024 20:39:37 +0100 Subject: [PATCH 17/25] Check if the plugins config file exists --- cmd/run_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/run_test.go b/cmd/run_test.go index f0b6a481..35e1c676 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -178,8 +178,8 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { } func Test_runCmdWithCachePlugin(t *testing.T) { - globalTestConfigFile := "./test_global_runCmdWithCachePlugin.yaml" - pluginTestConfigFile := "./test_plugins_runCmdWithCachePlugin.yaml" + globalTestConfigFile := "test_global_runCmdWithCachePlugin.yaml" + pluginTestConfigFile := "test_plugins_runCmdWithCachePlugin.yaml" // TODO: Remove this once these global variables are removed from cmd/run.go. // https://github.com/gatewayd-io/gatewayd/issues/324 stopChan = make(chan struct{}) @@ -210,6 +210,8 @@ func Test_runCmdWithCachePlugin(t *testing.T) { assert.Contains(t, output, "Plugin binary extracted to plugins/gatewayd-plugin-cache") assert.Contains(t, output, "Plugin installed successfully") + require.FileExists(t, pluginTestConfigFile, "plugin install command should have created a config file") + // See if the plugin was actually installed. output, err = executeCommandC(rootCmd, "plugin", "list", "-p", pluginTestConfigFile) require.NoError(t, err, "plugin list should not return an error") From 00423f84093f637b03d2b436a01377c1794b0743 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sat, 27 Jan 2024 21:15:15 +0100 Subject: [PATCH 18/25] Use an integration token to prevent GitHub API rate limits from blocking builds --- .github/workflows/test.yaml | 2 ++ .gitignore | 3 ++- cmd/config_init_test.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b3f09902..3a407821 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,6 +59,8 @@ jobs: - name: Run Go tests 🔬 run: go test -p 1 -cover -covermode atomic -coverprofile=profile.cov -v ./... + env: + GITHUB_AUTH_TOKEN: ${{ secrets.INTEGRATION }} - name: Report coverage to coveralls 📈 uses: shogo82148/actions-goveralls@v1 diff --git a/.gitignore b/.gitignore index 0e597b2f..78b3074d 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ dist/ libtensorflow* # Test generated files -cmd/test_plugins.yaml.bak +cmd/test_*.yaml.bak +cmd/test_*.yaml diff --git a/cmd/config_init_test.go b/cmd/config_init_test.go index 456db54f..79e073c0 100644 --- a/cmd/config_init_test.go +++ b/cmd/config_init_test.go @@ -22,7 +22,7 @@ func Test_configInitCmd(t *testing.T) { assert.FileExists(t, globalTestConfigFile, "configInitCmd should create a config file") // Test configInitCmd with the --force flag to overwrite the config file. - output, err = executeCommandC(rootCmd, "config", "init", "--force") + output, err = executeCommandC(rootCmd, "config", "init", "--force", "-c", globalTestConfigFile) require.NoError(t, err, "configInitCmd should not return an error") assert.Equal(t, fmt.Sprintf("Config file '%s' was overwritten successfully.", globalTestConfigFile), From 3e06f08bab503d236f54a697c723d0c9bd21e731 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 29 Jan 2024 00:55:04 +0100 Subject: [PATCH 19/25] Pull plugin once and use the archive to install the plugin Fix bug in stopChan being closed --- cmd/cmd_helpers_test.go | 17 +++++++++++++++++ cmd/plugin_install_test.go | 17 ++++++++--------- cmd/run_test.go | 34 ++++++++++++++++++---------------- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/cmd/cmd_helpers_test.go b/cmd/cmd_helpers_test.go index e8ed00bc..a891a018 100644 --- a/cmd/cmd_helpers_test.go +++ b/cmd/cmd_helpers_test.go @@ -2,6 +2,8 @@ package cmd import ( "bytes" + "os" + "path/filepath" "github.com/spf13/cobra" ) @@ -18,3 +20,18 @@ func executeCommandC(root *cobra.Command, args ...string) (string, error) { return buf.String(), err } + +// mustPullPlugin pulls the gatewayd-plugin-cache plugin and returns the path to the archive. +func mustPullPlugin() (string, error) { + pluginURL := "github.com/gatewayd-io/gatewayd-plugin-cache@v0.2.10" + fileName := "./gatewayd-plugin-cache-linux-amd64-v0.2.10.tar.gz" + + if _, err := os.Stat(fileName); os.IsNotExist(err) { + _, err := executeCommandC(rootCmd, "plugin", "install", "--pull-only", pluginURL) + if err != nil { + return "", err + } + } + + return filepath.Abs(fileName) +} diff --git a/cmd/plugin_install_test.go b/cmd/plugin_install_test.go index 1af6dcba..afd08e05 100644 --- a/cmd/plugin_install_test.go +++ b/cmd/plugin_install_test.go @@ -21,18 +21,17 @@ func Test_pluginInstallCmd(t *testing.T) { "plugin init command should have returned the correct output") assert.FileExists(t, pluginTestConfigFile, "plugin init command should have created a config file") + // Pull the plugin archive and install it. + pluginArchivePath, err = mustPullPlugin() + require.NoError(t, err, "mustPullPlugin should not return an error") + assert.FileExists(t, pluginArchivePath, "mustPullPlugin should have downloaded the plugin archive") + // Test plugin install command. output, err = executeCommandC( - rootCmd, "plugin", "install", - "github.com/gatewayd-io/gatewayd-plugin-cache@v0.2.4", - "-p", pluginTestConfigFile, "--update", "--backup") + rootCmd, "plugin", "install", "-p", pluginTestConfigFile, + "--update", "--backup", "--name", "gatewayd-plugin-cache", pluginArchivePath) require.NoError(t, err, "plugin install should not return an error") - assert.Contains(t, output, "Downloading https://github.com/gatewayd-io/gatewayd-plugin-cache/releases/download/v0.2.4/gatewayd-plugin-cache-linux-amd64-v0.2.4.tar.gz") //nolint:lll - assert.Contains(t, output, "Downloading https://github.com/gatewayd-io/gatewayd-plugin-cache/releases/download/v0.2.4/checksums.txt") //nolint:lll - assert.Contains(t, output, "Download completed successfully") - assert.Contains(t, output, "Checksum verification passed") - assert.Contains(t, output, "Plugin binary extracted to plugins/gatewayd-plugin-cache") - assert.Contains(t, output, "Plugin installed successfully") + assert.Equal(t, output, "Installing plugin from CLI argument\nBackup completed successfully\nPlugin binary extracted to plugins/gatewayd-plugin-cache\nPlugin installed successfully\n") //nolint:lll // See if the plugin was actually installed. output, err = executeCommandC(rootCmd, "plugin", "list", "-p", pluginTestConfigFile) diff --git a/cmd/run_test.go b/cmd/run_test.go index 35e1c676..6487641f 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -12,7 +12,10 @@ import ( "github.com/zenizh/go-capturer" ) -var waitBeforeStop = time.Second +var ( + waitBeforeStop = time.Second + pluginArchivePath = "" +) func Test_runCmd(t *testing.T) { globalTestConfigFile := "./test_global_runCmd.yaml" @@ -28,6 +31,8 @@ func Test_runCmd(t *testing.T) { // Check that the config file was created. assert.FileExists(t, globalTestConfigFile, "configInitCmd should create a config file") + stopChan = make(chan struct{}) + var waitGroup sync.WaitGroup waitGroup.Add(1) @@ -178,8 +183,8 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { } func Test_runCmdWithCachePlugin(t *testing.T) { - globalTestConfigFile := "test_global_runCmdWithCachePlugin.yaml" - pluginTestConfigFile := "test_plugins_runCmdWithCachePlugin.yaml" + globalTestConfigFile := "./test_global_runCmdWithCachePlugin.yaml" + pluginTestConfigFile := "./test_plugins_runCmdWithCachePlugin.yaml" // TODO: Remove this once these global variables are removed from cmd/run.go. // https://github.com/gatewayd-io/gatewayd/issues/324 stopChan = make(chan struct{}) @@ -195,28 +200,25 @@ func Test_runCmdWithCachePlugin(t *testing.T) { // Check that the config file was created. assert.FileExists(t, globalTestConfigFile, "configInitCmd should create a config file") + // Pull the plugin archive and install it. + pluginArchivePath, err = mustPullPlugin() + require.NoError(t, err, "mustPullPlugin should not return an error") + assert.FileExists(t, pluginArchivePath, "mustPullPlugin should have downloaded the plugin archive") + // Test plugin install command. output, err := executeCommandC( - rootCmd, "plugin", "install", - "-p", pluginTestConfigFile, "--update", - "github.com/gatewayd-io/gatewayd-plugin-cache@latest") + rootCmd, "plugin", "install", "-p", pluginTestConfigFile, "--update", "--backup", + "--name", "gatewayd-plugin-cache", pluginArchivePath) require.NoError(t, err, "plugin install should not return an error") - assert.Contains(t, output, "Installing plugin from CLI argument") - assert.Contains(t, output, "Downloading ") - assert.Contains(t, output, "gatewayd-plugin-cache-linux-amd64-") - assert.Contains(t, output, "/checksums.txt") - assert.Contains(t, output, "Download completed successfully") - assert.Contains(t, output, "Checksum verification passed") - assert.Contains(t, output, "Plugin binary extracted to plugins/gatewayd-plugin-cache") - assert.Contains(t, output, "Plugin installed successfully") - - require.FileExists(t, pluginTestConfigFile, "plugin install command should have created a config file") + assert.Equal(t, output, "Installing plugin from CLI argument\nBackup completed successfully\nPlugin binary extracted to plugins/gatewayd-plugin-cache\nPlugin installed successfully\n") //nolint:lll // See if the plugin was actually installed. output, err = executeCommandC(rootCmd, "plugin", "list", "-p", pluginTestConfigFile) require.NoError(t, err, "plugin list should not return an error") assert.Contains(t, output, "Name: gatewayd-plugin-cache") + stopChan = make(chan struct{}) + var waitGroup sync.WaitGroup waitGroup.Add(1) From cf990c220d0eb987524d2cd1c9cc1e9725fb6b86 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 29 Jan 2024 00:57:25 +0100 Subject: [PATCH 20/25] Ignore linter error --- cmd/cmd_helpers_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cmd_helpers_test.go b/cmd/cmd_helpers_test.go index a891a018..50ff9427 100644 --- a/cmd/cmd_helpers_test.go +++ b/cmd/cmd_helpers_test.go @@ -33,5 +33,5 @@ func mustPullPlugin() (string, error) { } } - return filepath.Abs(fileName) + return filepath.Abs(fileName) //nolint:wrapcheck } From 842f360562da36415bbb0fc9eaa2730d6a848129 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 29 Jan 2024 01:09:11 +0100 Subject: [PATCH 21/25] Close reader --- cmd/run_test.go | 2 +- cmd/utils.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/run_test.go b/cmd/run_test.go index 6487641f..b9ae34bf 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -129,7 +129,7 @@ func Test_runCmdWithTLS(t *testing.T) { // Test_runCmdWithMultiTenancy tests the run command with multi-tenancy enabled. // Note: This test needs two instances of PostgreSQL running on ports 5432 and 5433. func Test_runCmdWithMultiTenancy(t *testing.T) { - globalTestConfigFile := "testdata/gatewayd.yaml" + globalTestConfigFile := "./testdata/gatewayd.yaml" pluginTestConfigFile := "./test_plugins_runCmdWithMultiTenancy.yaml" // Create a test plugins config file. _, err := executeCommandC(rootCmd, "plugin", "init", "--force", "-p", pluginTestConfigFile) diff --git a/cmd/utils.go b/cmd/utils.go index fdac79d1..550c62e2 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -294,6 +294,7 @@ func extractTarGz(filename, dest string) ([]string, error) { if err != nil { return nil, gerr.ErrExtractFailed.Wrap(err) } + defer uncompressedStream.Close() // Create the output directory if it doesn't exist. if err := os.MkdirAll(dest, FolderPermissions); err != nil { From 45c6d95cfd872d80cf37510c431f065ca993cc49 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 29 Jan 2024 01:13:05 +0100 Subject: [PATCH 22/25] Reset global variable --- cmd/plugin_install_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/plugin_install_test.go b/cmd/plugin_install_test.go index afd08e05..ff5c1444 100644 --- a/cmd/plugin_install_test.go +++ b/cmd/plugin_install_test.go @@ -56,6 +56,9 @@ func Test_pluginInstallCmd(t *testing.T) { func Test_pluginInstallCmdAutomatedNoOverwrite(t *testing.T) { pluginTestConfigFile := "./testdata/gatewayd_plugins.yaml" + // Reset the global variable. + pullOnly = false + // Test plugin install command. output, err := executeCommandC( rootCmd, "plugin", "install", From 618b231ab110a0599af60f4f0e6466bd83d2f19a Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 29 Jan 2024 01:42:24 +0100 Subject: [PATCH 23/25] Overwrite config --- cmd/run_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/run_test.go b/cmd/run_test.go index b9ae34bf..c62c3aeb 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -208,7 +208,7 @@ func Test_runCmdWithCachePlugin(t *testing.T) { // Test plugin install command. output, err := executeCommandC( rootCmd, "plugin", "install", "-p", pluginTestConfigFile, "--update", "--backup", - "--name", "gatewayd-plugin-cache", pluginArchivePath) + "--overwrite-config=true", "--name", "gatewayd-plugin-cache", pluginArchivePath) require.NoError(t, err, "plugin install should not return an error") assert.Equal(t, output, "Installing plugin from CLI argument\nBackup completed successfully\nPlugin binary extracted to plugins/gatewayd-plugin-cache\nPlugin installed successfully\n") //nolint:lll From dfbfef08d0cd25715841033f861ab9ffb09f2b47 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 29 Jan 2024 01:44:54 +0100 Subject: [PATCH 24/25] Remove temp files --- cmd/run_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/run_test.go b/cmd/run_test.go index c62c3aeb..438e2ec2 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -2,6 +2,8 @@ package cmd import ( "context" + "fmt" + "os" "sync" "testing" "time" @@ -70,6 +72,9 @@ func Test_runCmd(t *testing.T) { }(&waitGroup) waitGroup.Wait() + + require.NoError(t, os.Remove(globalTestConfigFile)) + require.NoError(t, os.Remove(pluginTestConfigFile)) } // Test_runCmdWithTLS tests the run command with TLS enabled on the server. @@ -124,6 +129,8 @@ func Test_runCmdWithTLS(t *testing.T) { }(&waitGroup) waitGroup.Wait() + + require.NoError(t, os.Remove(pluginTestConfigFile)) } // Test_runCmdWithMultiTenancy tests the run command with multi-tenancy enabled. @@ -180,6 +187,8 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { }(&waitGroup) waitGroup.Wait() + + require.NoError(t, os.Remove(pluginTestConfigFile)) } func Test_runCmdWithCachePlugin(t *testing.T) { @@ -256,4 +265,9 @@ func Test_runCmdWithCachePlugin(t *testing.T) { }(&waitGroup) waitGroup.Wait() + + require.NoError(t, os.RemoveAll("plugins/")) + require.NoError(t, os.Remove(globalTestConfigFile)) + require.NoError(t, os.Remove(pluginTestConfigFile)) + require.NoError(t, os.Remove(fmt.Sprintf("%s.bak", pluginTestConfigFile))) } From 8d7c7302873a38f5cc261f1a92cfc6b98629c7dc Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 29 Jan 2024 14:38:15 +0100 Subject: [PATCH 25/25] Clarify install flags --- cmd/plugin_install.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/plugin_install.go b/cmd/plugin_install.go index 5e01e9f2..1c33751e 100644 --- a/cmd/plugin_install.go +++ b/cmd/plugin_install.go @@ -202,9 +202,9 @@ func init() { pluginInstallCmd.Flags().BoolVar( &backupConfig, "backup", false, "Backup the plugins configuration file before installing the plugin") pluginInstallCmd.Flags().StringVarP( - &pluginName, "name", "n", "", "Name of the plugin") + &pluginName, "name", "n", "", "Name of the plugin (only for installing from archive files)") pluginInstallCmd.Flags().BoolVar( - &overwriteConfig, "overwrite-config", true, "Overwrite the existing plugins configuration file") + &overwriteConfig, "overwrite-config", true, "Overwrite the existing plugins configuration file (overrides --update, only used for installing from the plugins configuration file)") //nolint:lll pluginInstallCmd.Flags().BoolVar( &enableSentry, "sentry", true, "Enable Sentry") // Already exists in run.go }