Merge pull request #1050 from binwiederhier/message-size-limit+delay-limit

Message size limit+delay limit

Author: binwiederhier

cmd/serve.go

@@ -366,9 +366,9 @@ func printTier(c *cli.Context, tier *user.Tier) {
 	fmt.Fprintf(c.App.ErrWriter, "- Email limit: %d\n", tier.EmailLimit)
 	fmt.Fprintf(c.App.ErrWriter, "- Phone call limit: %d\n", tier.CallLimit)
 	fmt.Fprintf(c.App.ErrWriter, "- Reservation limit: %d\n", tier.ReservationLimit)
-	fmt.Fprintf(c.App.ErrWriter, "- Attachment file size limit: %s\n", util.FormatSize(tier.AttachmentFileSizeLimit))
-	fmt.Fprintf(c.App.ErrWriter, "- Attachment total size limit: %s\n", util.FormatSize(tier.AttachmentTotalSizeLimit))
+	fmt.Fprintf(c.App.ErrWriter, "- Attachment file size limit: %s\n", util.FormatSizeHuman(tier.AttachmentFileSizeLimit))
+	fmt.Fprintf(c.App.ErrWriter, "- Attachment total size limit: %s\n", util.FormatSizeHuman(tier.AttachmentTotalSizeLimit))
 	fmt.Fprintf(c.App.ErrWriter, "- Attachment expiry duration: %s (%d seconds)\n", tier.AttachmentExpiryDuration.String(), int64(tier.AttachmentExpiryDuration.Seconds()))
-	fmt.Fprintf(c.App.ErrWriter, "- Attachment daily bandwidth limit: %s\n", util.FormatSize(tier.AttachmentBandwidthLimit))
+	fmt.Fprintf(c.App.ErrWriter, "- Attachment daily bandwidth limit: %s\n", util.FormatSizeHuman(tier.AttachmentBandwidthLimit))
 	fmt.Fprintf(c.App.ErrWriter, "- Stripe prices (monthly/yearly): %s\n", prices)
 }

cmd/tier.go

@@ -738,9 +738,8 @@ Usage is pretty straight forward. You can set the delivery time using the `X-Del
 `3h`, `2 days`), or a natural language time string (e.g. `10am`, `8:30pm`, `tomorrow, 3pm`, `Tuesday, 7am`, 
 [and more](https://github.com/olebedev/when)). 
 
-As of today, the minimum delay you can set is **10 seconds** and the maximum delay is **3 days**. This can currently
-not be configured otherwise ([let me know](https://github.com/binwiederhier/ntfy/issues) if you'd like to change 
-these limits).
+As of today, the minimum delay you can set is **10 seconds** and the maximum delay is **3 days**. This can be configured
+with the `message-delay-limit` option).
 
 For the purposes of [message caching](config.md#message-cache), scheduled messages are kept in the cache until 12 hours 
 after they were delivered (or whatever the server-side cache duration is set to). For instance, if a message is scheduled

docs/config.md

@@ -13,7 +13,7 @@ Many thanks to [@tcaputi](https://github.com/tcaputi) for fixing the issues, and
 
 * UI not updating properly ([#267](https://github.com/binwiederhier/ntfy/issues/267)/[#402](https://github.com/binwiederhier/ntfy/issues/402), thanks to [@tcaputi](https://github.com/tcaputi))
 
-### ntfy server v2.8.0
+## ntfy server v2.8.0
 Released November 19, 2023
 
 This release brings a handful of random bug fixes: two unrelated access control list fixes, a fix around web app crashes for languages with underscores in the language code (e.g. `zh_Hant`, `zh_Hans`, `pt_BR`, ...), a workaround for the `Priority` header (often used in Cloudflare setups), and support among others support for HTML-only emails (finally), web app crash fixes 
@@ -1315,6 +1315,14 @@ and the [ntfy Android app](https://github.com/binwiederhier/ntfy-android/release
 
 ### ntfy server v2.9.0
 
+!!! info
+    **Breaking change**: The `Rate-Topics` header was removed due to a [DoS issue](https://github.com/binwiederhier/ntfy/issues/1048). This only affects installations with `visitor-subscriber-rate-limiting: true`, which is not the default and likely very rarely used.
+
+**Features:**
+
+* Support for larger message delays with `message-delay-limit` (see [message limits](config.md#message-limits), [#1050](https://github.com/binwiederhier/ntfy/pull/1050)/[#1019](https://github.com/binwiederhier/ntfy/issues/1019), thanks to [@MrChadMWood](https://github.com/MrChadMWood) for reporting)
+* Support for larger message body sizes with `message-size-limit` (use at your own risk, see [message limits](config.md#message-limits), [#836](https://github.com/binwiederhier/ntfy/pull/836)/[#1050](https://github.com/binwiederhier/ntfy/pull/1050), thanks to [@zhzy0077](https://github.com/zhzy0077) for implementing this, and to [@nkjshlsqja7331](https://github.com/nkjshlsqja7331) for reporting)
+
 **Bug fixes + maintenance:**
 
 * Remove `Rate-Topics` header due to DoS security issue if `visitor-subscriber-rate-limiting: true` ([#1048](https://github.com/binwiederhier/ntfy/issues/1048)) 

docs/publish.md

@@ -12,11 +12,12 @@ import (
 const (
 	DefaultListenHTTP                           = ":80"
 	DefaultCacheDuration                        = 12 * time.Hour
+	DefaultCacheBatchTimeout                    = time.Duration(0)
 	DefaultKeepaliveInterval                    = 45 * time.Second // Not too frequently to save battery (Android read timeout used to be 77s!)
 	DefaultManagerInterval                      = time.Minute
 	DefaultDelayedSenderInterval                = 10 * time.Second
-	DefaultMinDelay                             = 10 * time.Second
-	DefaultMaxDelay                             = 3 * 24 * time.Hour
+	DefaultMessageDelayMin                      = 10 * time.Second
+	DefaultMessageDelayMax                      = 3 * 24 * time.Hour
 	DefaultFirebaseKeepaliveInterval            = 3 * time.Hour    // ~control topic (Android), not too frequently to save battery
 	DefaultFirebasePollInterval                 = 20 * time.Minute // ~poll topic (iOS), max. 2-3 times per hour (see docs)
 	DefaultFirebaseQuotaExceededPenaltyDuration = 10 * time.Minute // Time that over-users are locked out of Firebase if it returns "quota exceeded"
@@ -34,7 +35,7 @@ const (
 // - total topic limit: max number of topics overall
 // - various attachment limits
 const (
-	DefaultMessageLengthLimit       = 4096 // Bytes
+	DefaultMessageSizeLimit         = 4096 // Bytes; note that FCM/APNS have a limit of ~4 KB for the entire message
 	DefaultTotalTopicLimit          = 15000
 	DefaultAttachmentTotalSizeLimit = int64(5 * 1024 * 1024 * 1024) // 5 GB
 	DefaultAttachmentFileSizeLimit  = int64(15 * 1024 * 1024)       // 15 MB
@@ -122,9 +123,9 @@ type Config struct {
 	MetricsEnable                        bool
 	MetricsListenHTTP                    string
 	ProfileListenHTTP                    string
-	MessageLimit                         int
-	MinDelay                             time.Duration
-	MaxDelay                             time.Duration
+	MessageDelayMin                      time.Duration
+	MessageDelayMax                      time.Duration
+	MessageSizeLimit                     int
 	TotalTopicLimit                      int
 	TotalAttachmentSizeLimit             int64
 	VisitorSubscriptionLimit             int
@@ -211,9 +212,9 @@ func NewConfig() *Config {
 		TwilioPhoneNumber:                    "",
 		TwilioVerifyBaseURL:                  "https://verify.twilio.com", // Override for tests
 		TwilioVerifyService:                  "",
-		MessageLimit:                         DefaultMessageLengthLimit,
-		MinDelay:                             DefaultMinDelay,
-		MaxDelay:                             DefaultMaxDelay,
+		MessageSizeLimit:                     DefaultMessageSizeLimit,
+		MessageDelayMin:                      DefaultMessageDelayMin,
+		MessageDelayMax:                      DefaultMessageDelayMax,
 		TotalTopicLimit:                      DefaultTotalTopicLimit,
 		TotalAttachmentSizeLimit:             0,
 		VisitorSubscriptionLimit:             DefaultVisitorSubscriptionLimit,

docs/releases.md

@@ -733,7 +733,7 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*message, e
 	if err != nil {
 		return nil, err
 	}
-	body, err := util.Peek(r.Body, s.config.MessageLimit)
+	body, err := util.Peek(r.Body, s.config.MessageSizeLimit)
 	if err != nil {
 		return nil, err
 	}
@@ -996,9 +996,9 @@ func (s *Server) parsePublishParams(r *http.Request, m *message) (cache bool, fi
 		delay, err := util.ParseFutureTime(delayStr, time.Now())
 		if err != nil {
 			return false, false, "", "", false, errHTTPBadRequestDelayCannotParse
-		} else if delay.Unix() < time.Now().Add(s.config.MinDelay).Unix() {
+		} else if delay.Unix() < time.Now().Add(s.config.MessageDelayMin).Unix() {
 			return false, false, "", "", false, errHTTPBadRequestDelayTooSmall
-		} else if delay.Unix() > time.Now().Add(s.config.MaxDelay).Unix() {
+		} else if delay.Unix() > time.Now().Add(s.config.MessageDelayMax).Unix() {
 			return false, false, "", "", false, errHTTPBadRequestDelayTooLarge
 		}
 		m.Time = delay.Unix()
@@ -1754,7 +1754,7 @@ func (s *Server) sendDelayedMessage(v *visitor, m *message) error {
 // before passing it on to the next handler. This is meant to be used in combination with handlePublish.
 func (s *Server) transformBodyJSON(next handleFunc) handleFunc {
 	return func(w http.ResponseWriter, r *http.Request, v *visitor) error {
-		m, err := readJSONWithLimit[publishMessage](r.Body, s.config.MessageLimit*2, false) // 2x to account for JSON format overhead
+		m, err := readJSONWithLimit[publishMessage](r.Body, s.config.MessageSizeLimit*2, false) // 2x to account for JSON format overhead
 		if err != nil {
 			return err
 		}
@@ -1812,7 +1812,7 @@ func (s *Server) transformBodyJSON(next handleFunc) handleFunc {
 
 func (s *Server) transformMatrixJSON(next handleFunc) handleFunc {
 	return func(w http.ResponseWriter, r *http.Request, v *visitor) error {
-		newRequest, err := newRequestFromMatrixJSON(r, s.config.BaseURL, s.config.MessageLimit)
+		newRequest, err := newRequestFromMatrixJSON(r, s.config.BaseURL, s.config.MessageSizeLimit)
 		if err != nil {
 			logvr(v, r).Tag(tagMatrix).Err(err).Debug("Invalid Matrix request")
 			if e, ok := err.(*errMatrixPushkeyRejected); ok {

server/config.go

@@ -236,6 +236,16 @@
 # upstream-base-url:
 # upstream-access-token:
 
+# Configures message-specific limits
+#
+# - message-size-limit defines the max size of a message body. Please note message sizes >4K are NOT RECOMMENDED,
+#   and largely untested. If FCM and/or APNS is used, the limit should stay 4K, because their limits are around that size.
+#   If you increase this size limit regardless, FCM and APNS will NOT work for large messages.
+# - message-delay-limit defines the max delay of a message when using the "Delay" header.
+#
+# message-size-limit: "4k"
+# message-delay-limit: "3d"
+
 # Rate limiting: Total number of topics before the server rejects new topics.
 #
 # global-topic-limit: 15000

server/server.go

@@ -718,11 +718,11 @@ func TestAccount_Reservation_Delete_Messages_And_Attachments(t *testing.T) {
 	require.Nil(t, s.userManager.AddUser("phil", "phil", user.RoleUser))
 	require.Nil(t, s.userManager.AddTier(&user.Tier{
 		Code:         "starter",
-		MessageLimit: 10,
+		MessageSizeLimit: 10,
 	}))
 	require.Nil(t, s.userManager.AddTier(&user.Tier{
 		Code:         "pro",
-		MessageLimit: 20,
+		MessageSizeLimit: 20,
 	}))
 	require.Nil(t, s.userManager.ChangeTier("phil", "starter"))
 

server/server.yml

@@ -150,8 +150,8 @@ func (s *smtpSession) Data(r io.Reader) error {
 			return err
 		}
 		body = strings.TrimSpace(body)
-		if len(body) > conf.MessageLimit {
-			body = body[:conf.MessageLimit]
+		if len(body) > conf.MessageSizeLimit {
+			body = body[:conf.MessageSizeLimit]
 		}
 		m := newDefaultMessage(s.topic, body)
 		subject := strings.TrimSpace(msg.Header.Get("Subject"))

server/server_account_test.go

@@ -30,10 +30,10 @@ const (
 	visitorDefaultCallsLimit = int64(0)
 )
 
-// Constants used to convert a tier-user's MessageLimit (see user.Tier) into adequate request limiter
+// Constants used to convert a tier-user's MessageSizeLimit (see user.Tier) into adequate request limiter
 // values (token bucket). This is only used to increase the values in server.yml, never decrease them.
 //
-// Example: Assuming a user.Tier's MessageLimit is 10,000:
+// Example: Assuming a user.Tier's MessageSizeLimit is 10,000:
 // - the allowed burst is 500 (= 10,000 * 5%), which is < 1000 (the max)
 // - the replenish rate is 2 * 10,000 / 24 hours
 const (

server/smtp_server.go

@@ -10,8 +10,8 @@ import (
 )
 
 var (
-	errUnparsableTime = errors.New("unable to parse time")
-	durationStrRegex  = regexp.MustCompile(`(?i)^(\d+)\s*(d|days?|h|hours?|m|mins?|minutes?|s|secs?|seconds?)$`)
+	errInvalidDuration = errors.New("unable to parse duration")
+	durationStrRegex   = regexp.MustCompile(`(?i)^(\d+)\s*(d|days?|h|hours?|m|mins?|minutes?|s|secs?|seconds?)$`)
 )
 
 const (
@@ -51,7 +51,7 @@ func ParseFutureTime(s string, now time.Time) (time.Time, error) {
 	if err == nil {
 		return t, nil
 	}
-	return time.Time{}, errUnparsableTime
+	return time.Time{}, errInvalidDuration
 }
 
 // ParseDuration is like time.ParseDuration, except that it also understands days (d), which
@@ -65,7 +65,7 @@ func ParseDuration(s string) (time.Duration, error) {
 	if matches != nil {
 		number, err := strconv.Atoi(matches[1])
 		if err != nil {
-			return 0, errUnparsableTime
+			return 0, errInvalidDuration
 		}
 		switch unit := matches[2][0:1]; unit {
 		case "d":
@@ -77,44 +77,62 @@ func ParseDuration(s string) (time.Duration, error) {
 		case "s":
 			return time.Duration(number) * time.Second, nil
 		default:
-			return 0, errUnparsableTime
+			return 0, errInvalidDuration
 		}
 	}
-	return 0, errUnparsableTime
+	return 0, errInvalidDuration
+}
+
+// FormatDuration formats a time.Duration into a human-readable string, e.g. "2d", "20h", "30m", "40s".
+// It rounds to the largest unit that is not zero, thereby effectively rounding down.
+func FormatDuration(d time.Duration) string {
+	if d >= 24*time.Hour {
+		return strconv.Itoa(int(d/(24*time.Hour))) + "d"
+	}
+	if d >= time.Hour {
+		return strconv.Itoa(int(d/time.Hour)) + "h"
+	}
+	if d >= time.Minute {
+		return strconv.Itoa(int(d/time.Minute)) + "m"
+	}
+	if d >= time.Second {
+		return strconv.Itoa(int(d/time.Second)) + "s"
+	}
+	return "0s"
 }
 
 func parseFromDuration(s string, now time.Time) (time.Time, error) {
 	d, err := ParseDuration(s)
 	if err == nil {
 		return now.Add(d), nil
 	}
-	return time.Time{}, errUnparsableTime
+	return time.Time{}, errInvalidDuration
 }
 
 func parseUnixTime(s string, now time.Time) (time.Time, error) {
 	t, err := strconv.Atoi(s)
 	if err != nil {
 		return time.Time{}, err
 	} else if int64(t) < now.Unix() {
-		return time.Time{}, errUnparsableTime
+		return time.Time{}, errInvalidDuration
 	}
 	return time.Unix(int64(t), 0).UTC(), nil
 }
 
 func parseNaturalTime(s string, now time.Time) (time.Time, error) {
 	r, err := when.EN.Parse(s, now) // returns "nil, nil" if no matches!
 	if err != nil || r == nil {
-		return time.Time{}, errUnparsableTime
+		return time.Time{}, errInvalidDuration
 	} else if r.Time.After(now) {
 		return r.Time, nil
 	}
 	// Hack: If the time is parsable, but not in the future,
 	// simply append "tomorrow, " to it.
 	r, err = when.EN.Parse("tomorrow, "+s, now) // returns "nil, nil" if no matches!
 	if err != nil || r == nil {
-		return time.Time{}, errUnparsableTime
+		return time.Time{}, errInvalidDuration
 	} else if r.Time.After(now) {
 		return r.Time, nil
 	}
-	return time.Time{}, errUnparsableTime
+	return time.Time{}, errInvalidDuration
 }

server/visitor.go

@@ -92,3 +92,27 @@ func TestParseDuration(t *testing.T) {
 	require.Nil(t, err)
 	require.Equal(t, time.Duration(0), d)
 }
+
+func TestFormatDuration(t *testing.T) {
+	values := []struct {
+		duration time.Duration
+		expected string
+	}{
+		{24 * time.Second, "24s"},
+		{56 * time.Minute, "56m"},
+		{time.Hour, "1h"},
+		{2 * time.Hour, "2h"},
+		{24 * time.Hour, "1d"},
+		{3 * 24 * time.Hour, "3d"},
+	}
+	for _, value := range values {
+		require.Equal(t, value.expected, FormatDuration(value.duration))
+		d, err := ParseDuration(FormatDuration(value.duration))
+		require.Nil(t, err)
+		require.Equalf(t, value.duration, d, "duration does not match: %v != %v", value.duration, d)
+	}
+}
+
+func TestFormatDuration_Rounded(t *testing.T) {
+	require.Equal(t, "1d", FormatDuration(47*time.Hour))
+}

util/time.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"math/rand"
 	"net/netip"
 	"os"
@@ -215,6 +216,8 @@ func ParseSize(s string) (int64, error) {
 		return -1, fmt.Errorf("cannot convert number %s", matches[1])
 	}
 	switch strings.ToUpper(matches[2]) {
+	case "T":
+		return int64(value) * 1024 * 1024 * 1024 * 1024, nil
 	case "G":
 		return int64(value) * 1024 * 1024 * 1024, nil
 	case "M":
@@ -226,8 +229,23 @@ func ParseSize(s string) (int64, error) {
 	}
 }
 
-// FormatSize formats bytes into a human-readable notation, e.g. 2.1 MB
+// FormatSize formats the size in a way that it can be parsed by ParseSize.
+// It does not include decimal places. Uneven sizes are rounded down.
 func FormatSize(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%d%c", int(math.Floor(float64(b)/float64(div))), "KMGT"[exp])
+}
+
+// FormatSizeHuman formats bytes into a human-readable notation, e.g. 2.1 MB
+func FormatSizeHuman(b int64) string {
 	const unit = 1024
 	if b < unit {
 		return fmt.Sprintf("%d bytes", b)
@@ -237,7 +255,7 @@ func FormatSize(b int64) string {
 		div *= unit
 		exp++
 	}
-	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
+	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGT"[exp])
 }
 
 // ReadPassword will read a password from STDIN. If the terminal supports it, it will not print the