pax_global_header00006660000000000000000000000064143100372750014514gustar00rootroot0000000000000052 comment=d8c5314ac6af926237f942ba5f73d3111bcf39b8 go-mbox-1.1.0/000077500000000000000000000000001431003727500130635ustar00rootroot00000000000000go-mbox-1.1.0/.build.yml000066400000000000000000000003061431003727500147620ustar00rootroot00000000000000image: alpine/edge packages: - go sources: - https://github.com/emersion/go-mbox tasks: - build: | cd go-mbox go build -v ./... - test: | cd go-mbox go test -v ./... go-mbox-1.1.0/LICENSE000066400000000000000000000020521431003727500140670ustar00rootroot00000000000000MIT License Copyright (c) 2019 Simon Ser Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. go-mbox-1.1.0/README.md000066400000000000000000000006301431003727500143410ustar00rootroot00000000000000# go-mbox **Forked from github.com/emersion/go-mbox** [![GoDoc](https://godoc.org/github.com/emersion/go-mbox?status.svg)](https://godoc.org/github.com/emersion/go-mbox) [![builds.sr.ht status](https://builds.sr.ht/~emersion/go-mbox/commits.svg)](https://builds.sr.ht/~emersion/go-mbox/commits?) Package mbox parses the mbox file format into messages and formats messages into mbox files. # License MIT go-mbox-1.1.0/go.mod000066400000000000000000000000561431003727500141720ustar00rootroot00000000000000module github.com/ProtonMail/go-mbox go 1.12 go-mbox-1.1.0/mbox.go000066400000000000000000000004441431003727500143610ustar00rootroot00000000000000// Package mbox parses and formats the mbox file format. // // As the mbox file format is not standardized this package expects the least // common denominator, the so called mboxo format. package mbox var ( header = []byte("From ") escapedHeader = append([]byte{'>'}, header...) ) go-mbox-1.1.0/reader.go000066400000000000000000000062131431003727500146560ustar00rootroot00000000000000// "THE BEER-WARE LICENSE" (Revision 42): // wrote this file. As long as you retain this notice // you can do whatever you want with this stuff. If we meet some day, and you // think this stuff is worth it, you can buy me a beer in return. // Tobias Rehbein package mbox import ( "bufio" "bytes" "errors" "io" "io/ioutil" ) // ErrInvalidFormat is the error returned by the NextMessage method of Reader if // its content is malformed in a way that it is not possible to extract a // message. var ErrInvalidFormat = errors.New("invalid mbox format") type messageReader struct { r *bufio.Reader next bytes.Buffer atEOF, atSeparator bool atMiddleOfLine bool lastDelimiter []byte } func (mr *messageReader) Read(p []byte) (int, error) { if mr.atEOF || mr.atSeparator { return 0, io.EOF } if mr.next.Len() == 0 { b, isPrefix, err := mr.r.ReadLine() if err != nil { mr.atEOF = true return 0, err } if !mr.atMiddleOfLine { if bytes.HasPrefix(b, header) { mr.atSeparator = true mr.lastDelimiter = b return 0, io.EOF } else if len(b) == 0 { // Check if the next line is separator. In such case the new // line should not be written to not have double new line. b, isPrefix, err = mr.r.ReadLine() if err != nil { mr.atEOF = true return 0, err } if bytes.HasPrefix(b, header) { mr.atSeparator = true mr.lastDelimiter = b return 0, io.EOF } mr.next.Write([]byte("\r\n")) } if bytes.HasPrefix(b, escapedHeader) { b = b[1:] } } mr.next.Write(b) if !isPrefix { mr.next.Write([]byte("\r\n")) } mr.atMiddleOfLine = isPrefix } return mr.next.Read(p) } // Reader reads an mbox archive. type Reader struct { r *bufio.Reader mr *messageReader lastDelimiter []byte } // NewReader returns a new Reader to read messages from mbox file format data // provided by io.Reader r. func NewReader(r io.Reader) *Reader { return &Reader{r: bufio.NewReader(r)} } // NextMessage returns the next message text (containing both the header and the // body). It will return io.EOF if there are no messages left. func (r *Reader) NextMessage() (io.Reader, error) { if r.mr == nil { for { b, isPrefix, err := r.r.ReadLine() if err != nil { return nil, err } isFromLine := bytes.HasPrefix(b, header) // Discard the rest of the line. for isPrefix { _, isPrefix, err = r.r.ReadLine() if err != nil { return nil, err } } if len(b) == 0 { continue } if isFromLine { r.lastDelimiter = append([]byte{}, b...) break } else { return nil, ErrInvalidFormat } } } else { if _, err := io.Copy(ioutil.Discard, r.mr); err != nil { return nil, err } r.lastDelimiter = append([]byte{}, r.mr.lastDelimiter...) if r.mr.atEOF { return nil, io.EOF } } r.mr = &messageReader{r: r.r} return r.mr, nil } // GetMessageDelimiter returns delimiter in mbox file after message was loaded. // Otherwise empty. func (r *Reader) GetMessageDelimiter() []byte { return r.lastDelimiter } go-mbox-1.1.0/reader_test.go000066400000000000000000000315351431003727500157220ustar00rootroot00000000000000package mbox import ( "bytes" "fmt" "io" "net/mail" "strings" "testing" ) const mboxWithOneMessage = `From herp.derp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test This is a simple test. And, by the way, this is how a "From" line is escaped in mboxo format: >From Herp Derp with love. Bye. ` const mboxWithThreeMessages = `From herp.derp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test This is a simple test. And, by the way, this is how a "From" line is escaped in mboxo format: >From Herp Derp with love. Bye. From derp.herp@example.com Thu Jan 1 00:00:01 2015 From: derp.herp@example.com (Derp Herp) Date: Thu, 02 Jan 2015 00:00:01 +0100 Subject: Another test This is another simple test. Another line. Bye. From bernd.lauert@example.com Thu Jan 3 00:00:01 2015 From: bernd.lauert@example.com (Bernd Lauert) Date: Thu, 03 Jan 2015 00:00:01 +0100 Subject: A last test This is the last simple test. Bye. ` const mboxWithStartingLF = ` From herp.derp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test This is a simple test. And, by the way, this is how a "From" line is escaped in mboxo format: >From Herp Derp with love. Bye. From derp.herp@example.com Thu Jan 1 00:00:01 2015 From: derp.herp@example.com (Derp Herp) Date: Thu, 02 Jan 2015 00:00:01 +0100 Subject: Another test This is another simple test. Another line. Bye. From bernd.lauert@example.com Thu Jan 3 00:00:01 2015 From: bernd.lauert@example.com (Bernd Lauert) Date: Thu, 03 Jan 2015 00:00:01 +0100 Subject: A last test This is the last simple test. Bye. ` const mboxWithThreeMessagesMalformedButValid = `From herp.derp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test This is a simple test. And, by the way, this is how a "From" line is escaped in mboxo format: >From Herp Derp with love. Bye. From derp.herp@example.com Thu Jan 1 00:00:01 2015 From: derp.herp@example.com (Derp Herp) Date: Thu, 02 Jan 2015 00:00:01 +0100 Subject: Another test This is another simple test. Another line. Bye. From bernd.lauert@example.com Thu Jan 3 00:00:01 2015 From: bernd.lauert@example.com (Bernd Lauert) Date: Thu, 03 Jan 2015 00:00:01 +0100 Subject: A last test This is the last simple test. Bye. ` const mboxWithOneMessageMissingSeparator = `From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test This is a simple test. And, by the way, this is how a "From" line is escaped in mboxo format: >From Herp Derp with love. Bye. ` const mboxFirstMessage = `From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test This is a simple test. And, by the way, this is how a "From" line is escaped in mboxo format: From Herp Derp with love. Bye. ` func toCRLF(s string) string { return strings.Replace(s, "\n", "\r\n", -1) } func testMboxMessage(t *testing.T, mbox string, count int) { b := bytes.NewBufferString(mbox) m := NewReader(b) for i := 0; i < count; i++ { r, err := m.NextMessage() if err != nil { t.Fatalf("Unexpected error after NextMessage(): %v", err) } var text bytes.Buffer _, err = text.ReadFrom(r) if err != nil { t.Errorf("Unexpected error reading message body: %v", err) } want := toCRLF(mboxFirstMessage) if i == 0 && text.String() != want { t.Errorf("Expected:\n %q\ngot\n%q", want, text.String()) } } if _, err := m.NextMessage(); err != io.EOF { t.Fatalf("Unexpected error after NextMessage(): %v", err) } } func TestMboxMessageWithOneMessage(t *testing.T) { testMboxMessage(t, mboxWithOneMessage, 1) } func TestMboxMessageWithThreeMessages(t *testing.T) { testMboxMessage(t, mboxWithThreeMessages, 3) } func TestMboxMessageWithStartingLF(t *testing.T) { testMboxMessage(t, mboxWithStartingLF, 3) } func TestMboxMessageWithThreeMessagesMalformedButValid(t *testing.T) { testMboxMessage(t, mboxWithThreeMessagesMalformedButValid, 3) } func testMboxMessageInvalid(t *testing.T, mbox string) { b := bytes.NewBufferString(mbox) m := NewReader(b) if _, err := m.NextMessage(); err == nil { t.Errorf("Missing error after Next(): %v", err) } } func TestMboxMessageWithOneMessageMissingSeparator(t *testing.T) { testMboxMessageInvalid(t, mboxWithOneMessageMissingSeparator) } func TestMboxMessageNoRead(t *testing.T) { b := bytes.NewBufferString(mboxWithThreeMessages) m := NewReader(b) n := 0 for { _, err := m.NextMessage() if err == io.EOF { break } else if err != nil { t.Fatalf("Unexpected error after NextMessage(): %v", err) } n++ } if _, err := m.NextMessage(); err != io.EOF { t.Fatalf("Unexpected error after NextMessage(): %v", err) } if n != 3 { t.Fatalf("Expected 3 mesages, got %v", n) } } // TODO: decide whether we should keep this test or not func DisabledTestScanMessageWithBoundaries(t *testing.T) { sourceData := ` From derp.herp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test Content-Type: multipart/alternative; boundary=Apple-Mail-D55D9B1A-A379-4D5C-BDA9-00D35DF424A0 This is a test of boundaries. Don't accept a new email via \nFrom until the boundary is done!' And, by the way, this is how a "From" line is escaped in mboxo format: From Herp Derp with love. From Herp Derp with love. Bye. --Apple-Mail-D55D9B1A-A379-4D5C-BDA9-00D35DF424A0-- From derp.herp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test This is the second email in a test of boundaries. ` expected := []string{ "This is a test of boundaries. Don't accept a new email via \\nFrom until the boundary is done!'\n\nAnd, by the way, this is how a \"From\" line is escaped in mboxo format:\nFrom Herp Derp with love.\n\nFrom Herp Derp with love.\n\nBye.\n--Apple-Mail-D55D9B1A-A379-4D5C-BDA9-00D35DF424A0--\n", "This is the second email in a test of boundaries.\n", } b := bytes.NewBufferString(sourceData) m := NewReader(b) for i := range expected { r, err := m.NextMessage() if err != nil { t.Fatalf("Unexpected error after NextMessage(): %v", err) } msg, err := mail.ReadMessage(r) if err != nil { t.Fatalf("mail.ReadMessage() = %v", err) } var body bytes.Buffer _, err = body.ReadFrom(msg.Body) if err != nil { t.Errorf("%d - Unexpected error reading message body: %v", i, err) continue } want := toCRLF(expected[i]) if body.String() != want { t.Errorf("%d - Expected:\n %q\ngot\n%q", i, want, body.String()) } } if _, err := m.NextMessage(); err != io.EOF { t.Errorf("Next() succeeded") } } func TestScanMessageWithTextBoundary(t *testing.T) { sourceData := ` From derp.herp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test Content-Type: text/html; charset="utf-8"; boundary="monkey_d3df4dc8-da5e-47dd-be15-f19c5ed55194" This is a test of boundaries. Don't accept a new email via \nFrom until the boundary is done!' And, by the way, this is how a "From" line is escaped in mboxo format: Bye. From derp.herp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test This is the second email in a test of boundaries. ` expected := []string{ "This is a test of boundaries. Don't accept a new email via \\nFrom until the boundary is done!'\n\nAnd, by the way, this is how a \"From\" line is escaped in mboxo format:\n\nBye.\n", "This is the second email in a test of boundaries.\n", } b := bytes.NewBufferString(sourceData) m := NewReader(b) for i := range expected { r, err := m.NextMessage() if err != nil { t.Fatalf("Unexpected error after NextMessage(): %v", err) } msg, err := mail.ReadMessage(r) if err != nil { t.Fatalf("mail.ReadMessage() = %v", err) } var body bytes.Buffer _, err = body.ReadFrom(msg.Body) if err != nil { t.Errorf("%d - Unexpected error reading message body: %v", i, err) continue } want := toCRLF(expected[i]) if body.String() != want { t.Errorf("%d - Expected:\n %q\ngot\n%q", i, want, body.String()) } } if _, err := m.NextMessage(); err != io.EOF { t.Errorf("Next() succeeded") } } func TestScanMessageWithBoundarySemicolon(t *testing.T) { mbox := `From notifications@github.com Tue Jun 7 05:46:46 2016 From: Sender To: foo/bar Message-ID: Subject: Re: [foo/bar] [question] Baz? (#1) Content-Type: multipart/alternative; boundary="--==_mimepart_5755da228145a_38da3facdf97329c42987b"; MIME-Version: 1.0 ----==_mimepart_5755da228145a_38da3facdf97329c42987b Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: 8bit Blah blah ----==_mimepart_5755da228145a_38da3facdf97329c42987b Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: 8bit

Blah blah

----==_mimepart_5755da228145a_38da3facdf97329c42987b-- From notifications@github.com Tue Jun 7 05:52:15 2016 From: Author To: frob/blab Message-ID: Subject: Re: [frob/blab] [question] Bling? (#1) Content-Type: multipart/alternative; boundary="--==_mimepart_5755db739a819_79783f996b0172c04025ee"; MIME-Version: 1.0 ----==_mimepart_5755db739a819_79783f996b0172c04025ee Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: 8bit Blah blah ----==_mimepart_5755db739a819_79783f996b0172c04025ee Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: 8bit

Blah blah

----==_mimepart_5755db739a819_79783f996b0172c04025ee-- ` expected := 2 b := bytes.NewBufferString(mbox) m := NewReader(b) n := 0 for { _, err := m.NextMessage() if err == io.EOF { break } else if err != nil { t.Fatalf("m.NextMessaage() = %v", err) } n += 1 } if n != expected { t.Errorf("Expected: %d; got: %d", expected, n) } } func TestReadingMessageWithLongLine(t *testing.T) { // We are testing longer line than what `bufio.MaxScanTokenSize` defines. // RFC 5322 section 2.1.1 says line MUST be less than 998 bytes long, but // some providers generates messages with such long lines. want := strings.Repeat("This is very long line.", 5000) // Over 100k bytes. mbox := fmt.Sprintf(`From herp.derp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test %s`, want) b := bytes.NewBufferString(mbox) m := NewReader(b) r, err := m.NextMessage() if err != nil { t.Fatalf("m.NextMessaage() = %v", err) } msg, err := mail.ReadMessage(r) if err != nil { t.Fatalf("mail.ReadMessage() = %v", err) } var body bytes.Buffer _, err = body.ReadFrom(msg.Body) if err != nil { t.Errorf("body.ReadFrom() = %v", err) } if body.String() != want+"\r\n" { t.Errorf("Expected:\n %q\ngot\n%q", want, body.String()) } _, err = m.NextMessage() if err != io.EOF { t.Fatalf("m.NextMessage() = %v", err) } } func TestReadDelimiter(t *testing.T) { wantDelimiters := []string{ "From herp.derp@example.com Thu Jan 1 00:00:01 2015", "From derp.herp@example.com Thu Jan 1 00:00:01 2015", "From bernd.lauert@example.com Thu Jan 3 00:00:01 2015", } b := bytes.NewBufferString(mboxWithThreeMessages) m := NewReader(b) for i := 0; i < len(wantDelimiters); i++ { _, err := m.NextMessage() if err != nil { t.Fatalf("Error should be nil but have %v", err) } if string(m.GetMessageDelimiter()) != wantDelimiters[i] { t.Errorf( "Message %d expected to have delimiter\n%q\nbut have\n%q", i, wantDelimiters[i], m.GetMessageDelimiter(), ) } } _, err := m.NextMessage() if err != io.EOF { t.Fatalf("Error should be EOF but have %v", err) } } func ExampleReader() { r := strings.NewReader(`From herp.derp@example.com Thu Jan 1 00:00:01 2015 From: herp.derp@example.com (Herp Derp) Date: Thu, 01 Jan 2015 00:00:01 +0100 Subject: Test This is a simple test. CU. From derp.herp@example.com Thu Jan 1 00:00:01 2015 From: derp.herp@example.com (Derp Herp) Date: Thu, 02 Jan 2015 00:00:01 +0100 Subject: Another test This is another simple test. Bye. `) mr := NewReader(r) for { r, err := mr.NextMessage() if err == io.EOF { break } else if err != nil { fmt.Print("Oops, something went wrong!", err) return } msg, err := mail.ReadMessage(r) if err != nil { fmt.Print("Oops, something went wrong!", err) return } fmt.Printf("Message from %v\n", msg.Header.Get("From")) } // Output: // Message from herp.derp@example.com (Herp Derp) // Message from derp.herp@example.com (Derp Herp) } go-mbox-1.1.0/writer.go000066400000000000000000000042331431003727500147300ustar00rootroot00000000000000package mbox import ( "bytes" "errors" "io" "time" ) type messageWriter struct { w io.Writer buf bytes.Buffer } func (mw *messageWriter) writeLine(l []byte) (int, error) { if bytes.HasPrefix(l, header) { if _, err := mw.w.Write([]byte{'>'}); err != nil { return 0, err } } return mw.w.Write(l) } func (mw *messageWriter) Write(p []byte) (int, error) { mw.buf.Write(p) b := mw.buf.Bytes() mw.buf.Reset() N := 0 for { i := bytes.IndexByte(b, '\n') if i < 0 { n, err := mw.buf.Write(b) N += n return N, err } var l []byte l, b = b[:i+1], b[i+1:] n := len(l) // Replace CRLF with LF if len(l) > 1 && l[len(l)-2] == '\r' { l = l[:len(l)-2] l = append(l, '\n') } _, err := mw.writeLine(l) N += n if err != nil { return N, err } } } func (mw *messageWriter) Close() error { b := mw.buf.Bytes() mw.buf.Reset() if _, err := mw.writeLine(b); err != nil { return err } _, err := mw.w.Write([]byte("\n\n")) return err } // Writer writes messages to a mbox stream. The Close method must be called to // end the stream. type Writer struct { w io.Writer last *messageWriter closed bool } // NewWriter creates a new Writer that writes messages to w. func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } // CreateMessage appends a message to the mbox stream. The message text // (including both the header and the body) should be written to the returned // io.Writer. func (w *Writer) CreateMessage(from string, t time.Time) (io.Writer, error) { if w.closed { return nil, errors.New("mbox: Writer.CreateMessage called after Close") } if w.last != nil { if err := w.last.Close(); err != nil { return nil, err } w.last = nil } if from == "" { from = "???@???" } if t.IsZero() { t = time.Now() } date := t.UTC().Format(time.ANSIC) line := "From " + from + " " + date + "\n" if _, err := io.WriteString(w.w, line); err != nil { return nil, err } w.last = &messageWriter{w: w.w} return w.last, nil } func (w *Writer) Close() error { if w.closed { return errors.New("mbox: Writer already closed") } w.closed = true if w.last != nil { return w.last.Close() } return nil } go-mbox-1.1.0/writer_test.go000066400000000000000000000027441431003727500157740ustar00rootroot00000000000000package mbox import ( "bytes" "io" "strings" "testing" "time" ) type testMessage struct { date string text string } func testWriter(t *testing.T, messages []testMessage) string { var b bytes.Buffer wc := NewWriter(&b) for _, m := range messages { r := strings.NewReader(m.text) date, _ := time.Parse(time.RFC1123Z, m.date) mw, err := wc.CreateMessage("", date) if err != nil { t.Fatal(err) } _, err = io.Copy(mw, r) if err != nil { t.Fatal(err) } } if err := wc.Close(); err != nil { t.Fatal(err) } return b.String() } func TestWriter(t *testing.T) { messages := []testMessage{ { "Thu, 01 Jan 2015 00:00:01 +0100", `Date: Thu, 01 Jan 2015 00:00:01 +0100 This is a simple test. And, by the way, this is how a "From" line is escaped in mboxo format: From Herp Derp with love. Bye.`, }, { "Thu, 02 Jan 2015 00:00:01 +0100", `Date: Thu, 02 Jan 2015 00:00:01 +0100` + "\r" + ` ` + "\r" + ` This is another simple test.` + "\r" + ` ` + "\r" + ` Another line.` + "\r" + ` ` + "\r" + ` Bye.`, }, } expected := `From ???@??? Wed Dec 31 23:00:01 2014 Date: Thu, 01 Jan 2015 00:00:01 +0100 This is a simple test. And, by the way, this is how a "From" line is escaped in mboxo format: >From Herp Derp with love. Bye. From ???@??? Thu Jan 1 23:00:01 2015 Date: Thu, 02 Jan 2015 00:00:01 +0100 This is another simple test. Another line. Bye. ` s := testWriter(t, messages) if s != expected { t.Error("Invalid mbox output:", s) } }