Disable TLS checks and Bugfix Shodan (#25)

* article update

* do not verify tls

* bugfix shodan scanner

---------

Co-authored-by: frjcomp <frj1@securelogon.ch>

Author: frjcomp

docs/article.md

@@ -19,6 +19,8 @@ Sweet now you have access to the GitLab instance with an account.
 The first thing to look out for: What projects do I have access to? Is it more than unauthenticated? 
 Some companies grant their developers `developer` access to each repository, this might become interesting.
 
+The main question: Is the access concept based on the least privilege principle?
+
 # Misconfigurations And Mishandling
 
 ## Secret Detection in Source Code

src/pipeleak/cmd/scan.go

@@ -28,7 +28,6 @@ func NewScanCmd() *cobra.Command {
 		log.Fatal().Stack().Err(err).Msg("Unable to require gitlab flag")
 	}
 
-	//@todo test null vs empty string when no account
 	scanCmd.Flags().StringVarP(&options.GitlabApiToken, "token", "t", "", "GitLab API Token")
 	err = scanCmd.MarkFlagRequired("token")
 	if err != nil {

src/pipeleak/cmd/shodan.go

@@ -3,17 +3,15 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
 	"encoding/json"
 	"io"
-	"net/http"
 	"net/url"
 	"os"
 	"path"
 	"strconv"
 	"strings"
-	"time"
 
+	"github.com/CompassSecurity/pipeleak/helper"
 	"github.com/perimeterx/marshmallow"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
@@ -32,7 +30,7 @@ type result struct {
 	Hostnames []string `json:"hostnames"`
 	Port      int      `json:"port"`
 	IPString  string   `json:"ip_str"`
-	Shodan    shodan   `json:"shodan"`
+	Shodan    shodan   `json:"_shodan"`
 }
 
 func NewShodanCmd() *cobra.Command {
@@ -63,7 +61,7 @@ func Shodan(cmd *cobra.Command, args []string) {
 
 	data, _ := io.ReadAll(jsonFile)
 	ctx := context.Background()
-	group := parallel.Unlimited(ctx)
+	group := parallel.Limited(ctx, 4)
 	ctr := 0
 
 	for _, line := range bytes.Split(data, []byte{'\n'}) {
@@ -74,9 +72,9 @@ func Shodan(cmd *cobra.Command, args []string) {
 			log.Error().Stack().Err(err).Msg("failed unmarshalling jsonl line")
 		} else {
 
-			isHttps := true
-			if strings.EqualFold("http", d.Shodan.Module) {
-				isHttps = false
+			isHttps := false
+			if strings.EqualFold("https", d.Shodan.Module) {
+				isHttps = true
 			}
 
 			if len(d.Hostnames) == 0 {
@@ -126,10 +124,7 @@ func isRegistrationEnabled(base string) (bool, error) {
 	u.Path = path.Join(u.Path, "/users/somenotexistigusr/exists")
 	s := u.String()
 
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	client := &http.Client{Transport: tr, Timeout: 15 * time.Second}
+	client := helper.GetNonVerifyingHTTPClient()
 	res, err := client.Get(s)
 
 	if err != nil {
@@ -160,10 +155,8 @@ func checkNrPublicRepos(base string) (int, error) {
 	if err != nil {
 		return 0, err
 	}
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	client := &http.Client{Transport: tr, Timeout: 15 * time.Second}
+
+	client := helper.GetNonVerifyingHTTPClient()
 	u.Path = "/api/v4/projects"
 	s := u.String()
 	res, err := client.Get(s + "?per_page=100")

src/pipeleak/gitlab/runner.go

@@ -3,6 +3,7 @@ package gitlab
 import (
 	"strings"
 
+	"github.com/CompassSecurity/pipeleak/helper"
 	"github.com/rs/zerolog/log"
 	"github.com/xanzy/go-gitlab"
 )
@@ -14,7 +15,7 @@ type runnerResult struct {
 }
 
 func ListAllAvailableRunners(gitlabUrl string, apiToken string) {
-	git, err := gitlab.NewClient(apiToken, gitlab.WithBaseURL(gitlabUrl))
+	git, err := helper.GetGitlabClient(apiToken, gitlabUrl)
 	if err != nil {
 		log.Fatal().Stack().Err(err).Msg("failed creating gitlab client")
 	}

src/pipeleak/helper/helper.go

@@ -53,7 +53,7 @@ func CookieSessionValid(gitlabUrl string, cookieVal string) {
 		return
 	}
 	req.AddCookie(&http.Cookie{Name: "_gitlab_session", Value: cookieVal})
-	client := &http.Client{}
+	client := GetNonVerifyingHTTPClient()
 	resp, err := client.Do(req)
 	if err != nil {
 		log.Fatal().Stack().Err(err).Msg("Failed GitLab session test")
@@ -71,7 +71,7 @@ func CookieSessionValid(gitlabUrl string, cookieVal string) {
 
 func DetermineVersion(gitlabUrl string, apiToken string) *gitlab.Version {
 	if len(apiToken) > 0 {
-		git, err := gitlab.NewClient(apiToken, gitlab.WithBaseURL(gitlabUrl))
+		git, err := GetGitlabClient(apiToken, gitlabUrl)
 		if err != nil {
 			return &gitlab.Version{Version: "none", Revision: "none"}
 		}
@@ -88,10 +88,7 @@ func DetermineVersion(gitlabUrl string, apiToken string) *gitlab.Version {
 		}
 		u.Path = path.Join(u.Path, "/help")
 
-		tr := &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		}
-		client := &http.Client{Transport: tr, Timeout: 15 * time.Second}
+		client := GetNonVerifyingHTTPClient()
 		response, err := client.Get(u.String())
 
 		if err != nil {
@@ -190,3 +187,15 @@ func RegisterGracefulShutdownHandler(handler ShutdownHandler) {
 	}()
 
 }
+
+func GetNonVerifyingHTTPClient() *http.Client {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	return &http.Client{Transport: tr, Timeout: 15 * time.Second}
+}
+
+func GetGitlabClient(token string, url string) (*gitlab.Client, error) {
+	client, err := gitlab.NewClient(token, gitlab.WithBaseURL(url), gitlab.WithHTTPClient(GetNonVerifyingHTTPClient()))
+	return client, err
+}

src/pipeleak/scanner/pipeline.go

@@ -131,7 +131,7 @@ func cleanUp() {
 func fetchProjects(options *ScanOptions) {
 	log.Info().Msg("Fetching projects")
 
-	git, err := gitlab.NewClient(options.GitlabApiToken, gitlab.WithBaseURL(options.GitlabUrl))
+	git, err := helper.GetGitlabClient(options.GitlabApiToken, options.GitlabUrl)
 	if err != nil {
 		log.Fatal().Stack().Err(err).Msg("failed creating gitlab client")
 	}
@@ -294,7 +294,7 @@ func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string,
 
 	req.AddCookie(&http.Cookie{Name: "_gitlab_session", Value: cookieVal})
 
-	client := &http.Client{}
+	client := helper.GetNonVerifyingHTTPClient()
 	resp, err := client.Do(req)
 	if err != nil {
 		log.Debug().Stack().Err(err).Msg("Failed requesting dotenv artifact")