Skip to content

storage/minio: Add options to disable signature and multipart for Minio Client #21780

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion modules/setting/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,10 @@ func getStorage(rootCfg ConfigProvider, name, typ string, targetSec *ini.Section
sec.Key("MINIO_BUCKET").MustString("gitea")
sec.Key("MINIO_LOCATION").MustString("us-east-1")
sec.Key("MINIO_USE_SSL").MustBool(false)
sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)

sec.Key("MINIO_DISABLE_SIGNATURE").MustBool(false)
sec.Key("MINIO_DISABLE_MULTIPART").MustBool(false)
sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)

if targetSec == nil {
targetSec, _ = rootCfg.NewSection(name)
Expand Down
32 changes: 30 additions & 2 deletions modules/storage/minio.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package storage

import (
"bytes"
"context"
"crypto/tls"
"io"
Expand Down Expand Up @@ -45,14 +46,17 @@ const MinioStorageType Type = "minio"

// MinioStorageConfig represents the configuration for a minio storage
type MinioStorageConfig struct {

Endpoint string `ini:"MINIO_ENDPOINT"`
AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"`
SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"`
Bucket string `ini:"MINIO_BUCKET"`
Location string `ini:"MINIO_LOCATION"`
BasePath string `ini:"MINIO_BASE_PATH"`
UseSSL bool `ini:"MINIO_USE_SSL"`
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
DisableSignature bool `ini:"MINIO_DISABLE_SIGNATURE"`
DisableMultipart bool `ini:"MINIO_DISABLE_MULTIPART"`
}

// MinioStorage returns a minio bucket storage
Expand All @@ -61,6 +65,7 @@ type MinioStorage struct {
client *minio.Client
bucket string
basePath string
config *MinioStorageConfig
}

func convertMinioErr(err error) error {
Expand Down Expand Up @@ -117,6 +122,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
client: minioClient,
bucket: config.Bucket,
basePath: config.BasePath,
config: &config,
}, nil
}

Expand All @@ -136,13 +142,35 @@ func (m *MinioStorage) Open(path string) (Object, error) {

// Save save a file to minio
func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) {
disableSignature, disableMultipart := false, false
if m.config != nil {
disableSignature, disableMultipart = m.config.DisableSignature, m.config.DisableMultipart
}

if disableMultipart && size < 0 {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So how to handle when size == 0?

// Attempts to read everything from the source into memory. This can take a big toll on memory, and it can become a potential DoS source
// but since we have disabled multipart upload this mean we can't really stream write anymore...
// well, unless we have a better way to estimate the stream size, this would be a workaround

buf := &bytes.Buffer{}
n, err := io.Copy(buf, r)
if err != nil {
// I guess this would likely be EOF or OOM...?
return -1, err
}

// Since we read all the data from the source, it might not be usable again,
// so we should swap the reader location to our memory buffer
r, size = buf, n
}

uploadInfo, err := m.client.PutObject(
m.ctx,
m.bucket,
m.buildMinioPath(path),
r,
size,
minio.PutObjectOptions{ContentType: "application/octet-stream"},
minio.PutObjectOptions{ContentType: "application/octet-stream", DisableContentSha256: disableSignature, DisableMultipart: disableMultipart},
)
if err != nil {
return 0, convertMinioErr(err)
Expand Down