pax_global_header00006660000000000000000000000064147014360150014513gustar00rootroot0000000000000052 comment=66e7e07bdde5690508bacd6e131a6abef17464ab golang-github-edsrzf-mmap-go-1.2.0/000077500000000000000000000000001470143601500170705ustar00rootroot00000000000000golang-github-edsrzf-mmap-go-1.2.0/.github/000077500000000000000000000000001470143601500204305ustar00rootroot00000000000000golang-github-edsrzf-mmap-go-1.2.0/.github/workflows/000077500000000000000000000000001470143601500224655ustar00rootroot00000000000000golang-github-edsrzf-mmap-go-1.2.0/.github/workflows/build-test.yml000066400000000000000000000006561470143601500252730ustar00rootroot00000000000000name: Build and Test on: [push] jobs: run: runs-on: ${{ matrix.operating-system }} strategy: matrix: operating-system: [ubuntu-latest, windows-latest, macos-latest] steps: - name: Checkout uses: actions/checkout@v2 - name: Setup Go uses: actions/setup-go@v2 with: go-version: '~1.19.0' - name: Build run: go build . - name: Test run: go test ./... golang-github-edsrzf-mmap-go-1.2.0/.gitignore000066400000000000000000000001041470143601500210530ustar00rootroot00000000000000*.out *.5 *.6 *.8 *.swp _obj _test testdata /.idea *.iml /notes.txt golang-github-edsrzf-mmap-go-1.2.0/LICENSE000066400000000000000000000027561470143601500201070ustar00rootroot00000000000000Copyright (c) 2011, Evan Shaw All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang-github-edsrzf-mmap-go-1.2.0/README.md000066400000000000000000000017301470143601500203500ustar00rootroot00000000000000mmap-go ======= ![Build Status](https://github.com/edsrzf/mmap-go/actions/workflows/build-test.yml/badge.svg) [![Go Reference](https://pkg.go.dev/badge/github.com/edsrzf/mmap-go.svg)](https://pkg.go.dev/github.com/edsrzf/mmap-go) mmap-go is a portable mmap package for the [Go programming language](http://golang.org). Operating System Support ======================== This package is tested using GitHub Actions on Linux, macOS, and Windows. It should also work on other Unix-like platforms, but hasn't been tested with them. I'm interested to hear about the results. This package compiles for Plan 9 and WebAssembly, but its functions always return errors. Related functions such as `mprotect` and `mincore` aren't included. I haven't found a way to implement them on Windows without introducing significant complexity. If you're running on a Unix-like platform and really need these features, it should still be possible to implement them on top of this package via `syscall`. golang-github-edsrzf-mmap-go-1.2.0/example_test.go000066400000000000000000000023331470143601500221120ustar00rootroot00000000000000package mmap_test import ( "fmt" "log" "os" "github.com/edsrzf/mmap-go" ) func ExampleMapRegion() { m, err := mmap.MapRegion(nil, 100, mmap.RDWR, mmap.ANON, 0) if err != nil { log.Fatal(err) } // m acts as a writable slice of bytes that is not managed by the Go runtime. fmt.Println(len(m)) // Because the region is not managed by the Go runtime, the Unmap method should // be called when finished with it to avoid leaking memory. if err := m.Unmap(); err != nil { log.Fatal(err) } // Output: 100 } func ExampleMap() { f, err := os.OpenFile("notes.txt", os.O_RDWR|os.O_CREATE, 0755) if err != nil { log.Fatal(err) } _, err = f.WriteString("Hello, world") if err != nil { log.Fatal(err) } // The file must be closed, even after calling Unmap. defer f.Close() m, err := mmap.Map(f, mmap.RDWR, 0) if err != nil { log.Fatal(err) } // m acts as a writable slice of bytes that is a view into the open file, notes.txt. // It is sized to the file contents automatically. fmt.Println(string(m)) // The Unmap method should be called when finished with it to avoid leaking memory // and to ensure that writes are flushed to disk. if err := m.Unmap(); err != nil { log.Fatal(err) } // Hello, world } golang-github-edsrzf-mmap-go-1.2.0/go.mod000066400000000000000000000001471470143601500202000ustar00rootroot00000000000000module github.com/edsrzf/mmap-go go 1.17 require golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang-github-edsrzf-mmap-go-1.2.0/go.sum000066400000000000000000000006361470143601500202300ustar00rootroot00000000000000golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6 h1:IcgEB62HYgAhX0Nd/QrVgZlxlcyxbGQHElLUhW2X4Fo= golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang-github-edsrzf-mmap-go-1.2.0/mmap.go000066400000000000000000000071051470143601500203540ustar00rootroot00000000000000// Copyright 2011 Evan Shaw. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // This file defines the common package interface and contains a little bit of // factored out logic. // Package mmap allows mapping files into memory. It tries to provide a simple, reasonably portable interface, // but doesn't go out of its way to abstract away every little platform detail. // This specifically means: // * forked processes may or may not inherit mappings // * a file's timestamp may or may not be updated by writes through mappings // * specifying a size larger than the file's actual size can increase the file's size // * If the mapped file is being modified by another process while your program's running, don't expect consistent results between platforms package mmap import ( "errors" "os" "reflect" "unsafe" ) const ( // RDONLY maps the memory read-only. // Attempts to write to the MMap object will result in undefined behavior. RDONLY = 0 // RDWR maps the memory as read-write. Writes to the MMap object will update the // underlying file. RDWR = 1 << iota // COPY maps the memory as copy-on-write. Writes to the MMap object will affect // memory, but the underlying file will remain unchanged. COPY // If EXEC is set, the mapped memory is marked as executable. EXEC ) const ( // If the ANON flag is set, the mapped memory will not be backed by a file. ANON = 1 << iota ) // MMap represents a file mapped into memory. type MMap []byte // Map maps an entire file into memory. // If ANON is set in flags, f is ignored. func Map(f *os.File, prot, flags int) (MMap, error) { return MapRegion(f, -1, prot, flags, 0) } // MapRegion maps part of a file into memory. // The offset parameter must be a multiple of the system's page size. // If length < 0, the entire file will be mapped. // If ANON is set in flags, f is ignored. func MapRegion(f *os.File, length int, prot, flags int, offset int64) (MMap, error) { if offset%int64(os.Getpagesize()) != 0 { return nil, errors.New("offset parameter must be a multiple of the system's page size") } var fd uintptr if flags&ANON == 0 { fd = uintptr(f.Fd()) if length < 0 { fi, err := f.Stat() if err != nil { return nil, err } length = int(fi.Size()) } } else { if length <= 0 { return nil, errors.New("anonymous mapping requires non-zero length") } fd = ^uintptr(0) } return mmap(length, uintptr(prot), uintptr(flags), fd, offset) } func (m *MMap) header() *reflect.SliceHeader { return (*reflect.SliceHeader)(unsafe.Pointer(m)) } func (m *MMap) addrLen() (uintptr, uintptr) { header := m.header() return header.Data, uintptr(header.Len) } // Lock keeps the mapped region in physical memory, ensuring that it will not be // swapped out. func (m MMap) Lock() error { return m.lock() } // Unlock reverses the effect of Lock, allowing the mapped region to potentially // be swapped out. // If m is already unlocked, aan error will result. func (m MMap) Unlock() error { return m.unlock() } // Flush synchronizes the mapping's contents to the file's contents on disk. func (m MMap) Flush() error { return m.flush() } // Unmap deletes the memory mapped region, flushes any remaining changes, and sets // m to nil. // Trying to read or write any remaining references to m after Unmap is called will // result in undefined behavior. // Unmap should only be called on the slice value that was originally returned from // a call to Map. Calling Unmap on a derived slice may cause errors. func (m *MMap) Unmap() error { err := m.unmap() *m = nil return err } golang-github-edsrzf-mmap-go-1.2.0/mmap_plan9.go000066400000000000000000000010061470143601500214510ustar00rootroot00000000000000// Copyright 2020 Evan Shaw. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package mmap import "syscall" func mmap(len int, inprot, inflags, fd uintptr, off int64) ([]byte, error) { return nil, syscall.EPLAN9 } func (m MMap) flush() error { return syscall.EPLAN9 } func (m MMap) lock() error { return syscall.EPLAN9 } func (m MMap) unlock() error { return syscall.EPLAN9 } func (m MMap) unmap() error { return syscall.EPLAN9 } golang-github-edsrzf-mmap-go-1.2.0/mmap_test.go000066400000000000000000000071761470143601500214230ustar00rootroot00000000000000// Copyright 2011 Evan Shaw. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // These tests are adapted from gommap: http://labix.org/gommap // Copyright (c) 2010, Gustavo Niemeyer package mmap import ( "bytes" "io/ioutil" "os" "path/filepath" "testing" ) var testData = []byte("0123456789ABCDEF") var testPath = filepath.Join(os.TempDir(), "testdata") func init() { f := openFile(os.O_RDWR | os.O_CREATE | os.O_TRUNC) f.Write(testData) f.Close() } func openFile(flags int) *os.File { f, err := os.OpenFile(testPath, flags, 0644) if err != nil { panic(err.Error()) } return f } func TestUnmap(t *testing.T) { f := openFile(os.O_RDONLY) defer f.Close() mmap, err := Map(f, RDONLY, 0) if err != nil { t.Errorf("error mapping: %s", err) } if err := mmap.Unmap(); err != nil { t.Errorf("error unmapping: %s", err) } } func TestReadWrite(t *testing.T) { f := openFile(os.O_RDWR) defer f.Close() mmap, err := Map(f, RDWR, 0) if err != nil { t.Errorf("error mapping: %s", err) } defer mmap.Unmap() if !bytes.Equal(testData, mmap) { t.Errorf("mmap != testData: %q, %q", mmap, testData) } mmap[9] = 'X' mmap.Flush() fileData, err := ioutil.ReadAll(f) if err != nil { t.Errorf("error reading file: %s", err) } if !bytes.Equal(fileData, []byte("012345678XABCDEF")) { t.Errorf("file wasn't modified") } // leave things how we found them mmap[9] = '9' mmap.Flush() } func TestProtFlagsAndErr(t *testing.T) { f := openFile(os.O_RDONLY) defer f.Close() if _, err := Map(f, RDWR, 0); err == nil { t.Errorf("expected error") } } func TestFlags(t *testing.T) { f := openFile(os.O_RDWR) defer f.Close() mmap, err := Map(f, COPY, 0) if err != nil { t.Errorf("error mapping: %s", err) } defer mmap.Unmap() mmap[9] = 'X' mmap.Flush() fileData, err := ioutil.ReadAll(f) if err != nil { t.Errorf("error reading file: %s", err) } if !bytes.Equal(fileData, testData) { t.Errorf("file was modified") } } // Test that we can map files from non-0 offsets // The page size on most Unixes is 4KB, but on Windows it's 64KB func TestNonZeroOffset(t *testing.T) { const pageSize = 65536 // Create a 2-page sized file bigFilePath := filepath.Join(os.TempDir(), "nonzero") fileobj, err := os.OpenFile(bigFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { panic(err.Error()) } bigData := make([]byte, 2*pageSize, 2*pageSize) fileobj.Write(bigData) fileobj.Close() // Map the first page by itself fileobj, err = os.OpenFile(bigFilePath, os.O_RDONLY, 0) if err != nil { panic(err.Error()) } m, err := MapRegion(fileobj, pageSize, RDONLY, 0, 0) if err != nil { t.Errorf("error mapping file: %s", err) } m.Unmap() fileobj.Close() // Map the second page by itself fileobj, err = os.OpenFile(bigFilePath, os.O_RDONLY, 0) if err != nil { panic(err.Error()) } m, err = MapRegion(fileobj, pageSize, RDONLY, 0, pageSize) if err != nil { t.Errorf("error mapping file: %s", err) } err = m.Unmap() if err != nil { t.Error(err) } m, err = MapRegion(fileobj, pageSize, RDONLY, 0, 1) if err == nil { t.Error("expect error because offset is not multiple of page size") } fileobj.Close() } func TestAnonymousMapping(t *testing.T) { const size = 4 * 1024 // Make an anonymous region mem, err := MapRegion(nil, size, RDWR, ANON, 0) if err != nil { t.Fatalf("failed to allocate memory for buffer: %v", err) } // Check memory writable for i := 0; i < size; i++ { mem[i] = 0x55 } // And unmap it err = mem.Unmap() if err != nil { t.Fatalf("failed to unmap memory for buffer: %v", err) } } golang-github-edsrzf-mmap-go-1.2.0/mmap_unix.go000066400000000000000000000017731470143601500214240ustar00rootroot00000000000000// Copyright 2011 Evan Shaw. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build darwin dragonfly freebsd linux openbsd solaris netbsd package mmap import ( "golang.org/x/sys/unix" ) func mmap(len int, inprot, inflags, fd uintptr, off int64) ([]byte, error) { flags := unix.MAP_SHARED prot := unix.PROT_READ switch { case inprot© != 0: prot |= unix.PROT_WRITE flags = unix.MAP_PRIVATE case inprot&RDWR != 0: prot |= unix.PROT_WRITE } if inprot&EXEC != 0 { prot |= unix.PROT_EXEC } if inflags&ANON != 0 { flags |= unix.MAP_ANON } b, err := unix.Mmap(int(fd), off, len, prot, flags) if err != nil { return nil, err } return b, nil } func (m MMap) flush() error { return unix.Msync([]byte(m), unix.MS_SYNC) } func (m MMap) lock() error { return unix.Mlock([]byte(m)) } func (m MMap) unlock() error { return unix.Munlock([]byte(m)) } func (m MMap) unmap() error { return unix.Munmap([]byte(m)) } golang-github-edsrzf-mmap-go-1.2.0/mmap_wasm.go000066400000000000000000000010131470143601500213730ustar00rootroot00000000000000// Copyright 2024 Evan Shaw. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package mmap import "syscall" func mmap(len int, inprot, inflags, fd uintptr, off int64) ([]byte, error) { return nil, syscall.ENOTSUP } func (m MMap) flush() error { return syscall.ENOTSUP } func (m MMap) lock() error { return syscall.ENOTSUP } func (m MMap) unlock() error { return syscall.ENOTSUP } func (m MMap) unmap() error { return syscall.ENOTSUP } golang-github-edsrzf-mmap-go-1.2.0/mmap_windows.go000066400000000000000000000103461470143601500221270ustar00rootroot00000000000000// Copyright 2011 Evan Shaw. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package mmap import ( "errors" "os" "sync" "golang.org/x/sys/windows" ) // mmap on Windows is a two-step process. // First, we call CreateFileMapping to get a handle. // Then, we call MapviewToFile to get an actual pointer into memory. // Because we want to emulate a POSIX-style mmap, we don't want to expose // the handle -- only the pointer. We also want to return only a byte slice, // not a struct, so it's convenient to manipulate. // We keep this map so that we can get back the original handle from the memory address. type addrinfo struct { file windows.Handle mapview windows.Handle writable bool } var handleLock sync.Mutex var handleMap = map[uintptr]*addrinfo{} func mmap(len int, prot, flags, hfile uintptr, off int64) ([]byte, error) { flProtect := uint32(windows.PAGE_READONLY) dwDesiredAccess := uint32(windows.FILE_MAP_READ) writable := false switch { case prot© != 0: flProtect = windows.PAGE_WRITECOPY dwDesiredAccess = windows.FILE_MAP_COPY writable = true case prot&RDWR != 0: flProtect = windows.PAGE_READWRITE dwDesiredAccess = windows.FILE_MAP_WRITE writable = true } if prot&EXEC != 0 { flProtect <<= 4 dwDesiredAccess |= windows.FILE_MAP_EXECUTE } // The maximum size is the area of the file, starting from 0, // that we wish to allow to be mappable. It is the sum of // the length the user requested, plus the offset where that length // is starting from. This does not map the data into memory. maxSizeHigh := uint32((off + int64(len)) >> 32) maxSizeLow := uint32((off + int64(len)) & 0xFFFFFFFF) // TODO: Do we need to set some security attributes? It might help portability. h, errno := windows.CreateFileMapping(windows.Handle(hfile), nil, flProtect, maxSizeHigh, maxSizeLow, nil) if h == 0 { return nil, os.NewSyscallError("CreateFileMapping", errno) } // Actually map a view of the data into memory. The view's size // is the length the user requested. fileOffsetHigh := uint32(off >> 32) fileOffsetLow := uint32(off & 0xFFFFFFFF) addr, errno := windows.MapViewOfFile(h, dwDesiredAccess, fileOffsetHigh, fileOffsetLow, uintptr(len)) if addr == 0 { windows.CloseHandle(windows.Handle(h)) return nil, os.NewSyscallError("MapViewOfFile", errno) } handleLock.Lock() handleMap[addr] = &addrinfo{ file: windows.Handle(hfile), mapview: h, writable: writable, } handleLock.Unlock() m := MMap{} dh := m.header() dh.Data = addr dh.Len = len dh.Cap = dh.Len return m, nil } func (m MMap) flush() error { addr, len := m.addrLen() errno := windows.FlushViewOfFile(addr, len) if errno != nil { return os.NewSyscallError("FlushViewOfFile", errno) } handleLock.Lock() defer handleLock.Unlock() handle, ok := handleMap[addr] if !ok { // should be impossible; we would've errored above return errors.New("unknown base address") } if handle.writable && handle.file != windows.Handle(^uintptr(0)) { if err := windows.FlushFileBuffers(handle.file); err != nil { return os.NewSyscallError("FlushFileBuffers", err) } } return nil } func (m MMap) lock() error { addr, len := m.addrLen() errno := windows.VirtualLock(addr, len) return os.NewSyscallError("VirtualLock", errno) } func (m MMap) unlock() error { addr, len := m.addrLen() errno := windows.VirtualUnlock(addr, len) return os.NewSyscallError("VirtualUnlock", errno) } func (m MMap) unmap() error { err := m.flush() if err != nil { return err } addr := m.header().Data // Lock the UnmapViewOfFile along with the handleMap deletion. // As soon as we unmap the view, the OS is free to give the // same addr to another new map. We don't want another goroutine // to insert and remove the same addr into handleMap while // we're trying to remove our old addr/handle pair. handleLock.Lock() defer handleLock.Unlock() err = windows.UnmapViewOfFile(addr) if err != nil { return err } handle, ok := handleMap[addr] if !ok { // should be impossible; we would've errored above return errors.New("unknown base address") } delete(handleMap, addr) e := windows.CloseHandle(windows.Handle(handle.mapview)) return os.NewSyscallError("CloseHandle", e) }