diff --git a/cmd/serve.go b/cmd/serve.go index 5f97d421..3baf81ec 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -284,7 +284,7 @@ func execServe(c *cli.Context) error { } // Check values - if databaseURL != "" && (!strings.HasPrefix(databaseURL, "postgres://") || !strings.HasPrefix(databaseURL, "postgresql://") { + if databaseURL != "" && !strings.HasPrefix(databaseURL, "postgres://") && !strings.HasPrefix(databaseURL, "postgresql://") { return errors.New("if database-url is set, it must start with postgres:// or postgresql://") } else if databaseURL != "" && (authFile != "" || cacheFile != "" || webPushFile != "") { return errors.New("if database-url is set, auth-file, cache-file, and web-push-file must not be set") diff --git a/docs/releases.md b/docs/releases.md index 0acc5ac8..6641c580 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -6,12 +6,23 @@ and the [ntfy Android app](https://github.com/binwiederhier/ntfy-android/release | Component | Version | Release date | |------------------|---------|--------------| -| ntfy server | v2.19.0 | Mar 15, 2026 | +| ntfy server | v2.19.1 | Mar 15, 2026 | | ntfy Android app | v1.24.0 | Mar 5, 2026 | | ntfy iOS app | v1.3 | Nov 26, 2023 | Please check out the release notes for [upcoming releases](#not-released-yet) below. +## ntfy server v2.19.1 +Released March 15, 2026 + +This is a bugfix release to avoid PostgreSQL insert failures due to invalid UTF-8 messages. It also fixes `database-url` +validation incorrectly rejecting `postgresql://` connection strings. + +**Bug fixes + maintenance:** + +* Fix invalid UTF-8 in HTTP headers (e.g. Latin-1 encoded text) causing PostgreSQL insert failures and dropping entire message batches +* Fix `database-url` validation rejecting `postgresql://` connection strings ([#1657](https://github.com/binwiederhier/ntfy/issues/1657)/[#1658](https://github.com/binwiederhier/ntfy/pull/1658)) + ## ntfy server v2.19.0 Released March 15, 2026 diff --git a/message/cache.go b/message/cache.go index b123fba4..76aba4be 100644 --- a/message/cache.go +++ b/message/cache.go @@ -125,16 +125,16 @@ func (c *Cache) addMessages(ms []*model.Message) error { return model.ErrUnexpectedMessageType } published := m.Time <= time.Now().Unix() - tags := strings.Join(m.Tags, ",") + tags := util.SanitizeUTF8(strings.Join(m.Tags, ",")) var attachmentName, attachmentType, attachmentURL string var attachmentSize, attachmentExpires int64 var attachmentDeleted bool if m.Attachment != nil { - attachmentName = m.Attachment.Name - attachmentType = m.Attachment.Type + attachmentName = util.SanitizeUTF8(m.Attachment.Name) + attachmentType = util.SanitizeUTF8(m.Attachment.Type) attachmentSize = m.Attachment.Size attachmentExpires = m.Attachment.Expires - attachmentURL = m.Attachment.URL + attachmentURL = util.SanitizeUTF8(m.Attachment.URL) } var actionsStr string if len(m.Actions) > 0 { @@ -154,13 +154,13 @@ func (c *Cache) addMessages(ms []*model.Message) error { m.Time, m.Event, m.Expires, - m.Topic, - m.Message, - m.Title, + util.SanitizeUTF8(m.Topic), + util.SanitizeUTF8(m.Message), + util.SanitizeUTF8(m.Title), m.Priority, tags, - m.Click, - m.Icon, + util.SanitizeUTF8(m.Click), + util.SanitizeUTF8(m.Icon), actionsStr, attachmentName, attachmentType, @@ -170,7 +170,7 @@ func (c *Cache) addMessages(ms []*model.Message) error { attachmentDeleted, // Always zero sender, m.User, - m.ContentType, + util.SanitizeUTF8(m.ContentType), m.Encoding, published, ) diff --git a/message/cache_test.go b/message/cache_test.go index eb992381..0fddc88b 100644 --- a/message/cache_test.go +++ b/message/cache_test.go @@ -827,3 +827,141 @@ func TestStore_MessageFieldRoundTrip(t *testing.T) { require.Equal(t, `{"key":"value"}`, retrieved.Actions[1].Body) }) } + +func TestStore_AddMessage_InvalidUTF8(t *testing.T) { + forEachBackend(t, func(t *testing.T, s *message.Cache) { + // 0xc9 0x43: Latin-1 "ÉC" — 0xc9 starts a 2-byte UTF-8 sequence but 0x43 ('C') is not a continuation byte + m := model.NewDefaultMessage("mytopic", "\xc9Cas du serveur") + require.Nil(t, s.AddMessage(m)) + messages, err := s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, 1, len(messages)) + require.Equal(t, "\uFFFDCas du serveur", messages[0].Message) + + // 0xae: Latin-1 "®" — isolated byte above 0x7F, not a valid UTF-8 start for single byte + m2 := model.NewDefaultMessage("mytopic", "Product\xae Pro") + require.Nil(t, s.AddMessage(m2)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "Product\uFFFD Pro", messages[1].Message) + + // 0xe8 0x6d 0x65: Latin-1 "ème" — 0xe8 starts a 3-byte UTF-8 sequence but 0x6d ('m') is not a continuation byte + m3 := model.NewDefaultMessage("mytopic", "probl\xe8me critique") + require.Nil(t, s.AddMessage(m3)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "probl\uFFFDme critique", messages[2].Message) + + // 0xb2: Latin-1 "²" — isolated byte in 0x80-0xBF range (UTF-8 continuation byte without lead) + m4 := model.NewDefaultMessage("mytopic", "CO\xb2 level high") + require.Nil(t, s.AddMessage(m4)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "CO\uFFFD level high", messages[3].Message) + + // 0xe9 0x6d 0x61: Latin-1 "éma" — 0xe9 starts a 3-byte UTF-8 sequence but 0x6d ('m') is not a continuation byte + m5 := model.NewDefaultMessage("mytopic", "th\xe9matique") + require.Nil(t, s.AddMessage(m5)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "th\uFFFDmatique", messages[4].Message) + + // 0xed 0x64 0x65: Latin-1 "íde" — 0xed starts a 3-byte UTF-8 sequence but 0x64 ('d') is not a continuation byte + m6 := model.NewDefaultMessage("mytopic", "vid\xed\x64eo surveillance") + require.Nil(t, s.AddMessage(m6)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "vid\uFFFDdeo surveillance", messages[5].Message) + + // 0xf3 0x6e 0x3a 0x20: Latin-1 "ón: " — 0xf3 starts a 4-byte UTF-8 sequence but 0x6e ('n') is not a continuation byte + m7 := model.NewDefaultMessage("mytopic", "notificaci\xf3n: alerta") + require.Nil(t, s.AddMessage(m7)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "notificaci\uFFFDn: alerta", messages[6].Message) + + // 0xb7: Latin-1 "·" — isolated continuation byte + m8 := model.NewDefaultMessage("mytopic", "item\xb7value") + require.Nil(t, s.AddMessage(m8)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "item\uFFFDvalue", messages[7].Message) + + // 0xa8: Latin-1 "¨" — isolated continuation byte + m9 := model.NewDefaultMessage("mytopic", "na\xa8ve") + require.Nil(t, s.AddMessage(m9)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "na\uFFFDve", messages[8].Message) + + // 0xdf 0x64: Latin-1 "ßd" — 0xdf starts a 2-byte UTF-8 sequence but 0x64 ('d') is not a continuation byte + m10 := model.NewDefaultMessage("mytopic", "gro\xdf\x64ruck") + require.Nil(t, s.AddMessage(m10)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "gro\uFFFDdruck", messages[9].Message) + + // 0xe4 0x67 0x74: Latin-1 "ägt" — 0xe4 starts a 3-byte UTF-8 sequence but 0x67 ('g') is not a continuation byte + m11 := model.NewDefaultMessage("mytopic", "tr\xe4gt Last") + require.Nil(t, s.AddMessage(m11)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "tr\uFFFDgt Last", messages[10].Message) + + // 0xe9 0x65 0x20: Latin-1 "ée " — 0xe9 starts a 3-byte UTF-8 sequence but 0x65 ('e') is not a continuation byte + m12 := model.NewDefaultMessage("mytopic", "journ\xe9\x65 termin\xe9\x65") + require.Nil(t, s.AddMessage(m12)) + messages, err = s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, "journ\uFFFDe termin\uFFFDe", messages[11].Message) + }) +} + +func TestStore_AddMessage_NullByte(t *testing.T) { + forEachBackend(t, func(t *testing.T, s *message.Cache) { + // 0x00: NUL byte — valid UTF-8 but rejected by PostgreSQL + m := model.NewDefaultMessage("mytopic", "hello\x00world") + require.Nil(t, s.AddMessage(m)) + + messages, err := s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, 1, len(messages)) + require.Equal(t, "helloworld", messages[0].Message) + }) +} + +func TestStore_AddMessage_InvalidUTF8InTitleAndTags(t *testing.T) { + forEachBackend(t, func(t *testing.T, s *message.Cache) { + // Invalid UTF-8 can arrive via HTTP headers (Title, Tags) which bypass body validation + m := model.NewDefaultMessage("mytopic", "valid message") + m.Title = "\xc9clipse du syst\xe8me" + m.Tags = []string{"probl\xe8me", "syst\xe9me"} + m.Click = "https://example.com/\xae" + require.Nil(t, s.AddMessage(m)) + + messages, err := s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, 1, len(messages)) + require.Equal(t, "\uFFFDclipse du syst\uFFFDme", messages[0].Title) + require.Equal(t, "probl\uFFFDme", messages[0].Tags[0]) + require.Equal(t, "syst\uFFFDme", messages[0].Tags[1]) + require.Equal(t, "https://example.com/\uFFFD", messages[0].Click) + }) +} + +func TestStore_AddMessage_InvalidUTF8BatchDoesNotDropValidMessages(t *testing.T) { + forEachBackend(t, func(t *testing.T, s *message.Cache) { + // Previously, a single invalid message would roll back the entire batch transaction. + // Sanitization ensures all messages in a batch are written successfully. + msgs := []*model.Message{ + model.NewDefaultMessage("mytopic", "valid message 1"), + model.NewDefaultMessage("mytopic", "notificaci\xf3n: alerta"), + model.NewDefaultMessage("mytopic", "valid message 3"), + } + require.Nil(t, s.AddMessages(msgs)) + + messages, err := s.Messages("mytopic", model.SinceAllMessages, false) + require.Nil(t, err) + require.Equal(t, 3, len(messages)) + }) +} diff --git a/model/model.go b/model/model.go index a8ecdf78..97fecf2d 100644 --- a/model/model.go +++ b/model/model.go @@ -70,6 +70,26 @@ func (m *Message) Context() log.Context { return fields } +// SanitizeUTF8 replaces invalid UTF-8 sequences and strips NUL bytes from all user-supplied +// string fields. This is called early in the publish path so that all downstream consumers +// (Firebase, WebPush, SMTP, cache) receive clean UTF-8 strings. +func (m *Message) SanitizeUTF8() { + m.Topic = util.SanitizeUTF8(m.Topic) + m.Message = util.SanitizeUTF8(m.Message) + m.Title = util.SanitizeUTF8(m.Title) + m.Click = util.SanitizeUTF8(m.Click) + m.Icon = util.SanitizeUTF8(m.Icon) + m.ContentType = util.SanitizeUTF8(m.ContentType) + for i, tag := range m.Tags { + m.Tags[i] = util.SanitizeUTF8(tag) + } + if m.Attachment != nil { + m.Attachment.Name = util.SanitizeUTF8(m.Attachment.Name) + m.Attachment.Type = util.SanitizeUTF8(m.Attachment.Type) + m.Attachment.URL = util.SanitizeUTF8(m.Attachment.URL) + } +} + // ForJSON returns a copy of the message suitable for JSON output. // It clears the SequenceID if it equals the ID to reduce redundancy. func (m *Message) ForJSON() *Message { diff --git a/server/server.go b/server/server.go index 24c712bd..075d3079 100644 --- a/server/server.go +++ b/server/server.go @@ -880,6 +880,7 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*model.Mess if m.Message == "" { m.Message = emptyMessageBody } + m.SanitizeUTF8() delayed := m.Time > time.Now().Unix() ev := logvrm(v, r, m). Tag(tagPublish). diff --git a/server/server_test.go b/server/server_test.go index 24bf6cac..71743638 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -4441,3 +4441,88 @@ func TestServer_HandleError_SkipsWriteHeaderOnHijackedConnection(t *testing.T) { } }) } + +func TestServer_Publish_InvalidUTF8InBody(t *testing.T) { + // All byte sequences from production logs, sent as message body + tests := []struct { + name string + body string + message string + }{ + {"0xc9_0x43", "\xc9Cas du serveur", "\uFFFDCas du serveur"}, // Latin-1 "ÉC" + {"0xae", "Product\xae Pro", "Product\uFFFD Pro"}, // Latin-1 "®" + {"0xe8_0x6d_0x65", "probl\xe8me critique", "probl\uFFFDme critique"}, // Latin-1 "ème" + {"0xb2", "CO\xb2 level high", "CO\uFFFD level high"}, // Latin-1 "²" + {"0xe9_0x6d_0x61", "th\xe9matique", "th\uFFFDmatique"}, // Latin-1 "éma" + {"0xed_0x64_0x65", "vid\xed\x64eo surveillance", "vid\uFFFDdeo surveillance"}, // Latin-1 "íde" + {"0xf3_0x6e_0x3a_0x20", "notificaci\xf3n: alerta", "notificaci\uFFFDn: alerta"}, // Latin-1 "ón: " + {"0xb7", "item\xb7value", "item\uFFFDvalue"}, // Latin-1 "·" + {"0xa8", "na\xa8ve", "na\uFFFDve"}, // Latin-1 "¨" + {"0x00", "hello\x00world", "helloworld"}, // NUL byte + {"0xdf_0x64", "gro\xdf\x64ruck", "gro\uFFFDdruck"}, // Latin-1 "ßd" + {"0xe4_0x67_0x74", "tr\xe4gt Last", "tr\uFFFDgt Last"}, // Latin-1 "ägt" + {"0xe9_0x65_0x20", "journ\xe9\x65 termin\xe9\x65", "journ\uFFFDe termin\uFFFDe"}, // Latin-1 "ée" + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := newTestServer(t, newTestConfig(t, "")) + + // Publish via x-message header (the most common path for invalid UTF-8 from HTTP headers) + response := request(t, s, "PUT", "/mytopic", "", map[string]string{ + "X-Message": tc.body, + }) + require.Equal(t, 200, response.Code) + msg := toMessage(t, response.Body.String()) + require.Equal(t, tc.message, msg.Message) + + // Verify it was stored in the cache correctly + response = request(t, s, "GET", "/mytopic/json?poll=1", "", nil) + require.Equal(t, 200, response.Code) + msg = toMessage(t, response.Body.String()) + require.Equal(t, tc.message, msg.Message) + }) + } +} + +func TestServer_Publish_InvalidUTF8InTitle(t *testing.T) { + s := newTestServer(t, newTestConfig(t, "")) + response := request(t, s, "PUT", "/mytopic", "valid body", map[string]string{ + "Title": "\xc9clipse du syst\xe8me", + }) + require.Equal(t, 200, response.Code) + msg := toMessage(t, response.Body.String()) + require.Equal(t, "\uFFFDclipse du syst\uFFFDme", msg.Title) + require.Equal(t, "valid body", msg.Message) +} + +func TestServer_Publish_InvalidUTF8InTags(t *testing.T) { + s := newTestServer(t, newTestConfig(t, "")) + response := request(t, s, "PUT", "/mytopic", "valid body", map[string]string{ + "Tags": "probl\xe8me,syst\xe9me", + }) + require.Equal(t, 200, response.Code) + msg := toMessage(t, response.Body.String()) + require.Equal(t, "probl\uFFFDme", msg.Tags[0]) + require.Equal(t, "syst\uFFFDme", msg.Tags[1]) +} + +func TestServer_Publish_InvalidUTF8WithFirebase(t *testing.T) { + // Verify that sanitization happens before Firebase dispatch, so Firebase + // receives clean UTF-8 strings rather than invalid byte sequences + sender := newTestFirebaseSender(10) + s := newTestServer(t, newTestConfig(t, "")) + s.firebaseClient = newFirebaseClient(sender, &testAuther{Allow: true}) + + response := request(t, s, "PUT", "/mytopic", "", map[string]string{ + "X-Message": "notificaci\xf3n: alerta", + "Title": "\xc9clipse", + "Tags": "probl\xe8me", + }) + require.Equal(t, 200, response.Code) + + time.Sleep(100 * time.Millisecond) // Firebase publishing happens asynchronously + require.Equal(t, 1, len(sender.Messages())) + require.Equal(t, "notificaci\uFFFDn: alerta", sender.Messages()[0].Data["message"]) + require.Equal(t, "\uFFFDclipse", sender.Messages()[0].Data["title"]) + require.Equal(t, "probl\uFFFDme", sender.Messages()[0].Data["tags"]) +} diff --git a/util/util.go b/util/util.go index 85b2fbd4..be349691 100644 --- a/util/util.go +++ b/util/util.go @@ -17,6 +17,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "github.com/gabriel-vasile/mimetype" "golang.org/x/term" @@ -434,3 +435,22 @@ func Int(v int) *int { func Time(v time.Time) *time.Time { return &v } + +// SanitizeUTF8 ensures a string is safe to store in PostgreSQL by handling two cases: +// +// 1. Invalid UTF-8 sequences: Some clients send Latin-1/ISO-8859-1 encoded text (e.g. accented +// characters like é, ñ, ß) in HTTP headers or SMTP messages. Go treats these as raw bytes in +// strings, but PostgreSQL rejects them. Any invalid UTF-8 byte is replaced with the Unicode +// replacement character (U+FFFD, "�") so the message is still delivered rather than lost. +// +// 2. NUL bytes (0x00): These are valid in UTF-8 but PostgreSQL TEXT columns reject them. +// They are stripped entirely. +func SanitizeUTF8(s string) string { + if !utf8.ValidString(s) { + s = strings.ToValidUTF8(s, "\xef\xbf\xbd") // U+FFFD + } + if strings.ContainsRune(s, 0) { + s = strings.ReplaceAll(s, "\x00", "") + } + return s +}