BreizhHardware/ntfy
1
0
Fork 0
mirror of https://github.com/binwiederhier/ntfy.git synced 2026-03-18 21:30:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix UTF-8 insert failures in Postgres

Browse Source
This commit is contained in:
binwiederhier
2026-03-15 21:03:18 -04:00
parent 1afb99db67
commit 4699ed3ffd
8 changed files with 287 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/serve.go
View File

@@ -284,7 +284,7 @@ func execServe(c *cli.Context) error {
}
// Check values
if databaseURL != "" && (!strings.HasPrefix(databaseURL, "postgres://") || !strings.HasPrefix(databaseURL, "postgresql://") {
if databaseURL != "" && !strings.HasPrefix(databaseURL, "postgres://") && !strings.HasPrefix(databaseURL, "postgresql://") {
return errors.New("if database-url is set, it must start with postgres:// or postgresql://")
} else if databaseURL != "" && (authFile != "" || cacheFile != "" || webPushFile != "") {
return errors.New("if database-url is set, auth-file, cache-file, and web-push-file must not be set")

13
docs/releases.md
View File

@@ -6,12 +6,23 @@ and the [ntfy Android app](https://github.com/binwiederhier/ntfy-android/release
| Component | Version | Release date |
|------------------|---------|--------------|
| ntfy server | v2.19.0 | Mar 15, 2026 |
| ntfy server | v2.19.1 | Mar 15, 2026 |
| ntfy Android app | v1.24.0 | Mar 5, 2026 |
| ntfy iOS app | v1.3 | Nov 26, 2023 |
Please check out the release notes for [upcoming releases](#not-released-yet) below.
## ntfy server v2.19.1
Released March 15, 2026
This is a bugfix release to avoid PostgreSQL insert failures due to invalid UTF-8 messages. It also fixes `database-url`
validation incorrectly rejecting `postgresql://` connection strings.
**Bug fixes + maintenance:**
* Fix invalid UTF-8 in HTTP headers (e.g. Latin-1 encoded text) causing PostgreSQL insert failures and dropping entire message batches
* Fix `database-url` validation rejecting `postgresql://` connection strings ([#1657](https://github.com/binwiederhier/ntfy/issues/1657)/[#1658](https://github.com/binwiederhier/ntfy/pull/1658))
## ntfy server v2.19.0
Released March 15, 2026

20
message/cache.go
View File

@@ -125,16 +125,16 @@ func (c *Cache) addMessages(ms []*model.Message) error {
return model.ErrUnexpectedMessageType
}
published := m.Time <= time.Now().Unix()
tags := strings.Join(m.Tags, ",")
tags := util.SanitizeUTF8(strings.Join(m.Tags, ","))
var attachmentName, attachmentType, attachmentURL string
var attachmentSize, attachmentExpires int64
var attachmentDeleted bool
if m.Attachment != nil {
attachmentName = m.Attachment.Name
attachmentType = m.Attachment.Type
attachmentName = util.SanitizeUTF8(m.Attachment.Name)
attachmentType = util.SanitizeUTF8(m.Attachment.Type)
attachmentSize = m.Attachment.Size
attachmentExpires = m.Attachment.Expires
attachmentURL = m.Attachment.URL
attachmentURL = util.SanitizeUTF8(m.Attachment.URL)
}
var actionsStr string
if len(m.Actions) > 0 {
@@ -154,13 +154,13 @@ func (c *Cache) addMessages(ms []*model.Message) error {
m.Time,
m.Event,
m.Expires,
m.Topic,
m.Message,
m.Title,
util.SanitizeUTF8(m.Topic),
util.SanitizeUTF8(m.Message),
util.SanitizeUTF8(m.Title),
m.Priority,
tags,
m.Click,
m.Icon,
util.SanitizeUTF8(m.Click),
util.SanitizeUTF8(m.Icon),
actionsStr,
attachmentName,
attachmentType,
@@ -170,7 +170,7 @@ func (c *Cache) addMessages(ms []*model.Message) error {
attachmentDeleted, // Always zero
sender,
m.User,
m.ContentType,
util.SanitizeUTF8(m.ContentType),
m.Encoding,
published,
)

138
message/cache_test.go
View File

@@ -827,3 +827,141 @@ func TestStore_MessageFieldRoundTrip(t *testing.T) {
require.Equal(t, `{"key":"value"}`, retrieved.Actions[1].Body)
})
}
func TestStore_AddMessage_InvalidUTF8(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
// 0xc9 0x43: Latin-1 "ÉC" — 0xc9 starts a 2-byte UTF-8 sequence but 0x43 ('C') is not a continuation byte
m := model.NewDefaultMessage("mytopic", "\xc9Cas du serveur")
require.Nil(t, s.AddMessage(m))
messages, err := s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, 1, len(messages))
require.Equal(t, "\uFFFDCas du serveur", messages[0].Message)
// 0xae: Latin-1 "®" — isolated byte above 0x7F, not a valid UTF-8 start for single byte
m2 := model.NewDefaultMessage("mytopic", "Product\xae Pro")
require.Nil(t, s.AddMessage(m2))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "Product\uFFFD Pro", messages[1].Message)
// 0xe8 0x6d 0x65: Latin-1 "ème" — 0xe8 starts a 3-byte UTF-8 sequence but 0x6d ('m') is not a continuation byte
m3 := model.NewDefaultMessage("mytopic", "probl\xe8me critique")
require.Nil(t, s.AddMessage(m3))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "probl\uFFFDme critique", messages[2].Message)
// 0xb2: Latin-1 "²" — isolated byte in 0x80-0xBF range (UTF-8 continuation byte without lead)
m4 := model.NewDefaultMessage("mytopic", "CO\xb2 level high")
require.Nil(t, s.AddMessage(m4))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "CO\uFFFD level high", messages[3].Message)
// 0xe9 0x6d 0x61: Latin-1 "éma" — 0xe9 starts a 3-byte UTF-8 sequence but 0x6d ('m') is not a continuation byte
m5 := model.NewDefaultMessage("mytopic", "th\xe9matique")
require.Nil(t, s.AddMessage(m5))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "th\uFFFDmatique", messages[4].Message)
// 0xed 0x64 0x65: Latin-1 "íde" — 0xed starts a 3-byte UTF-8 sequence but 0x64 ('d') is not a continuation byte
m6 := model.NewDefaultMessage("mytopic", "vid\xed\x64eo surveillance")
require.Nil(t, s.AddMessage(m6))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "vid\uFFFDdeo surveillance", messages[5].Message)
// 0xf3 0x6e 0x3a 0x20: Latin-1 "ón: " — 0xf3 starts a 4-byte UTF-8 sequence but 0x6e ('n') is not a continuation byte
m7 := model.NewDefaultMessage("mytopic", "notificaci\xf3n: alerta")
require.Nil(t, s.AddMessage(m7))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "notificaci\uFFFDn: alerta", messages[6].Message)
// 0xb7: Latin-1 "·" — isolated continuation byte
m8 := model.NewDefaultMessage("mytopic", "item\xb7value")
require.Nil(t, s.AddMessage(m8))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "item\uFFFDvalue", messages[7].Message)
// 0xa8: Latin-1 "¨" — isolated continuation byte
m9 := model.NewDefaultMessage("mytopic", "na\xa8ve")
require.Nil(t, s.AddMessage(m9))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "na\uFFFDve", messages[8].Message)
// 0xdf 0x64: Latin-1 "ßd" — 0xdf starts a 2-byte UTF-8 sequence but 0x64 ('d') is not a continuation byte
m10 := model.NewDefaultMessage("mytopic", "gro\xdf\x64ruck")
require.Nil(t, s.AddMessage(m10))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "gro\uFFFDdruck", messages[9].Message)
// 0xe4 0x67 0x74: Latin-1 "ägt" — 0xe4 starts a 3-byte UTF-8 sequence but 0x67 ('g') is not a continuation byte
m11 := model.NewDefaultMessage("mytopic", "tr\xe4gt Last")
require.Nil(t, s.AddMessage(m11))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "tr\uFFFDgt Last", messages[10].Message)
// 0xe9 0x65 0x20: Latin-1 "ée " — 0xe9 starts a 3-byte UTF-8 sequence but 0x65 ('e') is not a continuation byte
m12 := model.NewDefaultMessage("mytopic", "journ\xe9\x65 termin\xe9\x65")
require.Nil(t, s.AddMessage(m12))
messages, err = s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, "journ\uFFFDe termin\uFFFDe", messages[11].Message)
})
}
func TestStore_AddMessage_NullByte(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
// 0x00: NUL byte — valid UTF-8 but rejected by PostgreSQL
m := model.NewDefaultMessage("mytopic", "hello\x00world")
require.Nil(t, s.AddMessage(m))
messages, err := s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, 1, len(messages))
require.Equal(t, "helloworld", messages[0].Message)
})
}
func TestStore_AddMessage_InvalidUTF8InTitleAndTags(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
// Invalid UTF-8 can arrive via HTTP headers (Title, Tags) which bypass body validation
m := model.NewDefaultMessage("mytopic", "valid message")
m.Title = "\xc9clipse du syst\xe8me"
m.Tags = []string{"probl\xe8me", "syst\xe9me"}
m.Click = "https://example.com/\xae"
require.Nil(t, s.AddMessage(m))
messages, err := s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, 1, len(messages))
require.Equal(t, "\uFFFDclipse du syst\uFFFDme", messages[0].Title)
require.Equal(t, "probl\uFFFDme", messages[0].Tags[0])
require.Equal(t, "syst\uFFFDme", messages[0].Tags[1])
require.Equal(t, "https://example.com/\uFFFD", messages[0].Click)
})
}
func TestStore_AddMessage_InvalidUTF8BatchDoesNotDropValidMessages(t *testing.T) {
forEachBackend(t, func(t *testing.T, s *message.Cache) {
// Previously, a single invalid message would roll back the entire batch transaction.
// Sanitization ensures all messages in a batch are written successfully.
msgs := []*model.Message{
model.NewDefaultMessage("mytopic", "valid message 1"),
model.NewDefaultMessage("mytopic", "notificaci\xf3n: alerta"),
model.NewDefaultMessage("mytopic", "valid message 3"),
}
require.Nil(t, s.AddMessages(msgs))
messages, err := s.Messages("mytopic", model.SinceAllMessages, false)
require.Nil(t, err)
require.Equal(t, 3, len(messages))
})
}

20
model/model.go
View File

@@ -70,6 +70,26 @@ func (m *Message) Context() log.Context {
return fields
}
// SanitizeUTF8 replaces invalid UTF-8 sequences and strips NUL bytes from all user-supplied
// string fields. This is called early in the publish path so that all downstream consumers
// (Firebase, WebPush, SMTP, cache) receive clean UTF-8 strings.
func (m *Message) SanitizeUTF8() {
m.Topic = util.SanitizeUTF8(m.Topic)
m.Message = util.SanitizeUTF8(m.Message)
m.Title = util.SanitizeUTF8(m.Title)
m.Click = util.SanitizeUTF8(m.Click)
m.Icon = util.SanitizeUTF8(m.Icon)
m.ContentType = util.SanitizeUTF8(m.ContentType)
for i, tag := range m.Tags {
m.Tags[i] = util.SanitizeUTF8(tag)
}
if m.Attachment != nil {
m.Attachment.Name = util.SanitizeUTF8(m.Attachment.Name)
m.Attachment.Type = util.SanitizeUTF8(m.Attachment.Type)
m.Attachment.URL = util.SanitizeUTF8(m.Attachment.URL)
}
}
// ForJSON returns a copy of the message suitable for JSON output.
// It clears the SequenceID if it equals the ID to reduce redundancy.
func (m *Message) ForJSON() *Message {

1
server/server.go
View File

@@ -880,6 +880,7 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*model.Mess
if m.Message == "" {
m.Message = emptyMessageBody
}
m.SanitizeUTF8()
delayed := m.Time > time.Now().Unix()
ev := logvrm(v, r, m).
Tag(tagPublish).

85
server/server_test.go
View File

@@ -4441,3 +4441,88 @@ func TestServer_HandleError_SkipsWriteHeaderOnHijackedConnection(t *testing.T) {
}
})
}
func TestServer_Publish_InvalidUTF8InBody(t *testing.T) {
// All byte sequences from production logs, sent as message body
tests := []struct {
name string
body string
message string
}{
{"0xc9_0x43", "\xc9Cas du serveur", "\uFFFDCas du serveur"}, // Latin-1 "ÉC"
{"0xae", "Product\xae Pro", "Product\uFFFD Pro"}, // Latin-1 "®"
{"0xe8_0x6d_0x65", "probl\xe8me critique", "probl\uFFFDme critique"}, // Latin-1 "ème"
{"0xb2", "CO\xb2 level high", "CO\uFFFD level high"}, // Latin-1 "²"
{"0xe9_0x6d_0x61", "th\xe9matique", "th\uFFFDmatique"}, // Latin-1 "éma"
{"0xed_0x64_0x65", "vid\xed\x64eo surveillance", "vid\uFFFDdeo surveillance"}, // Latin-1 "íde"
{"0xf3_0x6e_0x3a_0x20", "notificaci\xf3n: alerta", "notificaci\uFFFDn: alerta"}, // Latin-1 "ón: "
{"0xb7", "item\xb7value", "item\uFFFDvalue"}, // Latin-1 "·"
{"0xa8", "na\xa8ve", "na\uFFFDve"}, // Latin-1 "¨"
{"0x00", "hello\x00world", "helloworld"}, // NUL byte
{"0xdf_0x64", "gro\xdf\x64ruck", "gro\uFFFDdruck"}, // Latin-1 "ßd"
{"0xe4_0x67_0x74", "tr\xe4gt Last", "tr\uFFFDgt Last"}, // Latin-1 "ägt"
{"0xe9_0x65_0x20", "journ\xe9\x65 termin\xe9\x65", "journ\uFFFDe termin\uFFFDe"}, // Latin-1 "ée"
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
s := newTestServer(t, newTestConfig(t, ""))
// Publish via x-message header (the most common path for invalid UTF-8 from HTTP headers)
response := request(t, s, "PUT", "/mytopic", "", map[string]string{
"X-Message": tc.body,
})
require.Equal(t, 200, response.Code)
msg := toMessage(t, response.Body.String())
require.Equal(t, tc.message, msg.Message)
// Verify it was stored in the cache correctly
response = request(t, s, "GET", "/mytopic/json?poll=1", "", nil)
require.Equal(t, 200, response.Code)
msg = toMessage(t, response.Body.String())
require.Equal(t, tc.message, msg.Message)
})
}
}
func TestServer_Publish_InvalidUTF8InTitle(t *testing.T) {
s := newTestServer(t, newTestConfig(t, ""))
response := request(t, s, "PUT", "/mytopic", "valid body", map[string]string{
"Title": "\xc9clipse du syst\xe8me",
})
require.Equal(t, 200, response.Code)
msg := toMessage(t, response.Body.String())
require.Equal(t, "\uFFFDclipse du syst\uFFFDme", msg.Title)
require.Equal(t, "valid body", msg.Message)
}
func TestServer_Publish_InvalidUTF8InTags(t *testing.T) {
s := newTestServer(t, newTestConfig(t, ""))
response := request(t, s, "PUT", "/mytopic", "valid body", map[string]string{
"Tags": "probl\xe8me,syst\xe9me",
})
require.Equal(t, 200, response.Code)
msg := toMessage(t, response.Body.String())
require.Equal(t, "probl\uFFFDme", msg.Tags[0])
require.Equal(t, "syst\uFFFDme", msg.Tags[1])
}
func TestServer_Publish_InvalidUTF8WithFirebase(t *testing.T) {
// Verify that sanitization happens before Firebase dispatch, so Firebase
// receives clean UTF-8 strings rather than invalid byte sequences
sender := newTestFirebaseSender(10)
s := newTestServer(t, newTestConfig(t, ""))
s.firebaseClient = newFirebaseClient(sender, &testAuther{Allow: true})
response := request(t, s, "PUT", "/mytopic", "", map[string]string{
"X-Message": "notificaci\xf3n: alerta",
"Title": "\xc9clipse",
"Tags": "probl\xe8me",
})
require.Equal(t, 200, response.Code)
time.Sleep(100 * time.Millisecond) // Firebase publishing happens asynchronously
require.Equal(t, 1, len(sender.Messages()))
require.Equal(t, "notificaci\uFFFDn: alerta", sender.Messages()[0].Data["message"])
require.Equal(t, "\uFFFDclipse", sender.Messages()[0].Data["title"])
require.Equal(t, "probl\uFFFDme", sender.Messages()[0].Data["tags"])
}

20
util/util.go
View File

@@ -17,6 +17,7 @@ import (
"strings"
"sync"
"time"
"unicode/utf8"
"github.com/gabriel-vasile/mimetype"
"golang.org/x/term"
@@ -434,3 +435,22 @@ func Int(v int) *int {
func Time(v time.Time) *time.Time {
return &v
}
// SanitizeUTF8 ensures a string is safe to store in PostgreSQL by handling two cases:
//
// 1. Invalid UTF-8 sequences: Some clients send Latin-1/ISO-8859-1 encoded text (e.g. accented
// characters like é, ñ, ß) in HTTP headers or SMTP messages. Go treats these as raw bytes in
// strings, but PostgreSQL rejects them. Any invalid UTF-8 byte is replaced with the Unicode
// replacement character (U+FFFD, "<22>") so the message is still delivered rather than lost.
//
// 2. NUL bytes (0x00): These are valid in UTF-8 but PostgreSQL TEXT columns reject them.
// They are stripped entirely.
func SanitizeUTF8(s string) string {
if !utf8.ValidString(s) {
s = strings.ToValidUTF8(s, "\xef\xbf\xbd") // U+FFFD
}
if strings.ContainsRune(s, 0) {
s = strings.ReplaceAll(s, "\x00", "")
}
return s
}