phobos/std/mmfile.d

// Written in the D programming language.

/**
 * Read and write memory mapped files.
 * Macros:
 *  WIKI=Phobos/StdMmfile
 *
 * Copyright: Copyright Digital Mars 2004 - 2009.
 * License:   $(WEB www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
 * Authors:   $(WEB digitalmars.com, Walter Bright),
 *            Matthew Wilson
 * Source:    $(PHOBOSSRC std/_mmfile.d)
 */
/*          Copyright Digital Mars 2004 - 2009.
 * Distributed under the Boost Software License, Version 1.0.
 *    (See accompanying file LICENSE_1_0.txt or copy at
 *          http://www.boost.org/LICENSE_1_0.txt)
 */
module std.mmfile;

private import std.file;
private import core.stdc.stdio;
private import core.stdc.stdlib;
private import core.stdc.errno;
private import std.path;
private import std.string;
import std.conv, std.exception, std.stdio;

import std.internal.cstring;

//debug = MMFILE;

version (Windows)
{
    private import std.c.windows.windows;
    private import std.utf;
}
else version (Posix)
{
    private import core.sys.posix.fcntl;
    private import core.sys.posix.unistd;
    private import core.sys.posix.sys.mman;
    private import core.sys.posix.sys.stat;
}
else
{
    static assert(0);
}

/**
 * MmFile objects control the memory mapped file resource.
 */
class MmFile
{
    /**
     * The mode the memory mapped file is opened with.
     */
    enum Mode
    {
        read,            /// Read existing file
        readWriteNew,    /// Delete existing file, write new file
        readWrite,       /// Read/Write existing file, create if not existing
        readCopyOnWrite, /// Read/Write existing file, copy on write
    }

    /**
     * Open memory mapped file filename for reading.
     * File is closed when the object instance is deleted.
     * Throws:
     *  std.file.FileException
     */
    this(string filename)
    {
        this(filename, Mode.read, 0, null);
    }

    version(linux) this(File file, Mode mode = Mode.read, ulong size = 0,
            void* address = null, size_t window = 0)
    {
        this(file.fileno, mode, size, address, window);
    }

    version(linux) private this(int fildes, Mode mode, ulong size,
            void* address, size_t window)
    {
        int oflag;
        int fmode;

        switch (mode)
        {
        case Mode.read:
            flags = MAP_SHARED;
            prot = PROT_READ;
            oflag = O_RDONLY;
            fmode = 0;
            break;

        case Mode.readWriteNew:
            assert(size != 0);
            flags = MAP_SHARED;
            prot = PROT_READ | PROT_WRITE;
            oflag = O_CREAT | O_RDWR | O_TRUNC;
            fmode = octal!660;
            break;

        case Mode.readWrite:
            flags = MAP_SHARED;
            prot = PROT_READ | PROT_WRITE;
            oflag = O_CREAT | O_RDWR;
            fmode = octal!660;
            break;

        case Mode.readCopyOnWrite:
            flags = MAP_PRIVATE;
            prot = PROT_READ | PROT_WRITE;
            oflag = O_RDWR;
            fmode = 0;
            break;

        default:
            assert(0);
        }

        fd = fildes;

        // Adjust size
        stat_t statbuf = void;
        errnoEnforce(fstat(fd, &statbuf) == 0);
        if (prot & PROT_WRITE && size > statbuf.st_size)
        {
            // Need to make the file size bytes big
            lseek(fd, cast(off_t)(size - 1), SEEK_SET);
            char c = 0;
            core.sys.posix.unistd.write(fd, &c, 1);
        }
        else if (prot & PROT_READ && size == 0)
            size = statbuf.st_size;
        this.size = size;

        // Map the file into memory!
        size_t initial_map = (window && 2*window<size)
            ? 2*window : cast(size_t)size;
        auto p = mmap(address, initial_map, prot, flags, fd, 0);
        if (p == MAP_FAILED) {
            errnoEnforce(false, "Could not map file into memory");
        }
        data = p[0 .. initial_map];
    }

    /**
     * Open memory mapped file filename in mode.
     * File is closed when the object instance is deleted.
     * Params:
     *  filename = name of the file.
     *      If null, an anonymous file mapping is created.
     *  mode = access mode defined above.
     *  size =  the size of the file. If 0, it is taken to be the
     *      size of the existing file.
     *  address = the preferred address to map the file to,
     *      although the system is not required to honor it.
     *      If null, the system selects the most convenient address.
     *  window = preferred block size of the amount of data to map at one time
     *      with 0 meaning map the entire file. The window size must be a
     *      multiple of the memory allocation page size.
     * Throws:
     *  std.file.FileException
     */
    this(string filename, Mode mode, ulong size, void* address,
            size_t window = 0)
    {
        this.filename = filename;
        this.mMode = mode;
        this.window = window;
        this.address = address;

        version (Windows)
        {
            void* p;
            uint dwDesiredAccess2;
            uint dwShareMode;
            uint dwCreationDisposition;
            uint flProtect;

            switch (mode)
            {
            case Mode.read:
                dwDesiredAccess2 = GENERIC_READ;
                dwShareMode = FILE_SHARE_READ;
                dwCreationDisposition = OPEN_EXISTING;
                flProtect = PAGE_READONLY;
                dwDesiredAccess = FILE_MAP_READ;
                break;

            case Mode.readWriteNew:
                assert(size != 0);
                dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
                dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
                dwCreationDisposition = CREATE_ALWAYS;
                flProtect = PAGE_READWRITE;
                dwDesiredAccess = FILE_MAP_WRITE;
                break;

            case Mode.readWrite:
                dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
                dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
                dwCreationDisposition = OPEN_ALWAYS;
                flProtect = PAGE_READWRITE;
                dwDesiredAccess = FILE_MAP_WRITE;
                break;

            case Mode.readCopyOnWrite:
                dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
                dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
                dwCreationDisposition = OPEN_EXISTING;
                flProtect = PAGE_WRITECOPY;
                dwDesiredAccess = FILE_MAP_COPY;
                break;

            default:
                assert(0);
            }

            if (filename.ptr)
            {
                hFile = CreateFileW(filename.tempCStringW(),
                        dwDesiredAccess2,
                        dwShareMode,
                        null,
                        dwCreationDisposition,
                        FILE_ATTRIBUTE_NORMAL,
                        cast(HANDLE)null);
            }
            else
                hFile = null;

            hFileMap = null;
            if (hFile != INVALID_HANDLE_VALUE)
            {
                int hi = cast(int)(size>>32);
                hFileMap = CreateFileMappingW(hFile, null, flProtect,
                        hi, cast(uint)size, null);
            }
            if (hFileMap != null)               // mapping didn't fail
            {

                if (size == 0)
                {
                    uint sizehi;
                    uint sizelow = GetFileSize(hFile,&sizehi);
                    size = (cast(ulong)sizehi << 32) + sizelow;
                }
                this.size = size;

                size_t initial_map = (window && 2*window<size)
                    ? 2*window : cast(size_t)size;
                p = MapViewOfFileEx(hFileMap, dwDesiredAccess, 0, 0,
                        initial_map, address);
                if (p)
                {
                    data = p[0 .. initial_map];

                    debug (MMFILE) printf("MmFile.this(): p = %p, size = %d\n", p, size);
                    return;
                }
            }

            if (hFileMap != null)
                CloseHandle(hFileMap);
            hFileMap = null;

            if (hFile != INVALID_HANDLE_VALUE)
                CloseHandle(hFile);
            hFile = INVALID_HANDLE_VALUE;

            errnoEnforce(false);
        }
        else version (Posix)
        {
            void* p;
            int oflag;
            int fmode;

            switch (mode)
            {
            case Mode.read:
                flags = MAP_SHARED;
                prot = PROT_READ;
                oflag = O_RDONLY;
                fmode = 0;
                break;

            case Mode.readWriteNew:
                assert(size != 0);
                flags = MAP_SHARED;
                prot = PROT_READ | PROT_WRITE;
                oflag = O_CREAT | O_RDWR | O_TRUNC;
                fmode = octal!660;
                break;

            case Mode.readWrite:
                flags = MAP_SHARED;
                prot = PROT_READ | PROT_WRITE;
                oflag = O_CREAT | O_RDWR;
                fmode = octal!660;
                break;

            case Mode.readCopyOnWrite:
                flags = MAP_PRIVATE;
                prot = PROT_READ | PROT_WRITE;
                oflag = O_RDWR;
                fmode = 0;
                break;

            default:
                assert(0);
            }

            if (filename.length)
            {
                fd = .open(filename.tempCString(), oflag, fmode);
                errnoEnforce(fd != -1, "Could not open file "~filename);

                stat_t statbuf;
                if (fstat(fd, &statbuf))
                {
                    //printf("\tfstat error, errno = %d\n", errno);
                    .close(fd);
                    errnoEnforce(false, "Could not stat file "~filename);
                }

                if (prot & PROT_WRITE && size > statbuf.st_size)
                {
                    // Need to make the file size bytes big
                    .lseek(fd, cast(off_t)(size - 1), SEEK_SET);
                    char c = 0;
                    core.sys.posix.unistd.write(fd, &c, 1);
                }
                else if (prot & PROT_READ && size == 0)
                    size = statbuf.st_size;
            }
            else
            {
                fd = -1;
                version(linux) import core.sys.linux.sys.mman : MAP_ANON;                
                flags |= MAP_ANON;
            }
            this.size = size;
            size_t initial_map = (window && 2*window<size)
                ? 2*window : cast(size_t)size;
            p = mmap(address, initial_map, prot, flags, fd, 0);
            if (p == MAP_FAILED) {
                if (fd != -1)
                    .close(fd);
                errnoEnforce(fd != -1, "Could not map file "~filename);
            }

            data = p[0 .. initial_map];
        }
        else
        {
            static assert(0);
        }
    }

    /**
     * Flushes pending output and closes the memory mapped file.
     */
    ~this()
    {
        debug (MMFILE) printf("MmFile.~this()\n");
        unmap();
        version (Windows)
        {
            errnoEnforce(hFileMap == null || CloseHandle(hFileMap) == TRUE,
                    "Could not close file handle");
            hFileMap = null;

            errnoEnforce(!hFile || hFile == INVALID_HANDLE_VALUE
                    || CloseHandle(hFile) == TRUE,
                    "Could not close handle");
            hFile = INVALID_HANDLE_VALUE;
        }
        else version (Posix)
        {
            errnoEnforce(fd == -1 || fd <= 2
                    || .close(fd) != -1,
                    "Could not close handle");
            fd = -1;
        }
        else
        {
            static assert(0);
        }
        data = null;
    }

    /* Flush any pending output.
     */
    void flush()
    {
        debug (MMFILE) printf("MmFile.flush()\n");
        version (Windows)
        {
            FlushViewOfFile(data.ptr, data.length);
        }
        else version (Posix)
        {
            int i;
            i = msync(cast(void*)data, data.length, MS_SYNC);   // sys/mman.h
            errnoEnforce(i == 0, "msync failed");
        }
        else
        {
            static assert(0);
        }
    }

    /**
     * Gives size in bytes of the memory mapped file.
     */
    @property ulong length() const
    {
        debug (MMFILE) printf("MmFile.length()\n");
        return size;
    }

    /**
     * Read-only property returning the file mode.
     */
    Mode mode()
    {
        debug (MMFILE) printf("MmFile.mode()\n");
        return mMode;
    }

    /**
     * Returns entire file contents as an array.
     */
    void[] opSlice()
    {
        debug (MMFILE) printf("MmFile.opSlice()\n");
        return opSlice(0,size);
    }

    /**
     * Returns slice of file contents as an array.
     */
    void[] opSlice(ulong i1, ulong i2)
    {
        debug (MMFILE) printf("MmFile.opSlice(%lld, %lld)\n", i1, i2);
        ensureMapped(i1,i2);
        size_t off1 = cast(size_t)(i1-start);
        size_t off2 = cast(size_t)(i2-start);
        return data[off1 .. off2];
    }

    /**
     * Returns byte at index i in file.
     */
    ubyte opIndex(ulong i)
    {
        debug (MMFILE) printf("MmFile.opIndex(%lld)\n", i);
        ensureMapped(i);
        size_t off = cast(size_t)(i-start);
        return (cast(ubyte[])data)[off];
    }

    /**
     * Sets and returns byte at index i in file to value.
     */
    ubyte opIndexAssign(ubyte value, ulong i)
    {
        debug (MMFILE) printf("MmFile.opIndex(%lld, %d)\n", i, value);
        ensureMapped(i);
        size_t off = cast(size_t)(i-start);
        return (cast(ubyte[])data)[off] = value;
    }


    // return true if the given position is currently mapped
    private int mapped(ulong i)
    {
        debug (MMFILE) printf("MmFile.mapped(%lld, %lld, %d)\n", i,start,
                data.length);
        return i >= start && i < start+data.length;
    }

    // unmap the current range
    private void unmap()
    {
        debug (MMFILE) printf("MmFile.unmap()\n");
        version(Windows) {
            errnoEnforce(!data.ptr || UnmapViewOfFile(data.ptr) != FALSE);
        } else {
            errnoEnforce(!data.ptr || munmap(cast(void*)data, data.length) == 0,
                    "munmap failed");
        }
        data = null;
    }

    // map range
    private void map(ulong start, size_t len)
    {
        debug (MMFILE) printf("MmFile.map(%lld, %d)\n", start, len);
        void* p;
        if (start+len > size)
            len = cast(size_t)(size-start);
        version(Windows) {
            uint hi = cast(uint)(start>>32);
            p = MapViewOfFileEx(hFileMap, dwDesiredAccess, hi, cast(uint)start, len, address);
            errnoEnforce(p);
        } else {
            p = mmap(address, len, prot, flags, fd, cast(off_t)start);
            errnoEnforce(p != MAP_FAILED);
        }
        data = p[0 .. len];
        this.start = start;
    }

    // ensure a given position is mapped
    private void ensureMapped(ulong i)
    {
        debug (MMFILE) printf("MmFile.ensureMapped(%lld)\n", i);
        if (!mapped(i)) {
            unmap();
            if (window == 0) {
                map(0,cast(size_t)size);
            } else {
                ulong block = i/window;
                if (block == 0)
                    map(0,2*window);
                else
                    map(window*(block-1),3*window);
            }
        }
    }

    // ensure a given range is mapped
    private void ensureMapped(ulong i, ulong j)
    {
        debug (MMFILE) printf("MmFile.ensureMapped(%lld, %lld)\n", i, j);
        if (!mapped(i) || !mapped(j-1)) {
            unmap();
            if (window == 0) {
                map(0,cast(size_t)size);
            } else {
                ulong iblock = i/window;
                ulong jblock = (j-1)/window;
                if (iblock == 0) {
                    map(0,cast(size_t)(window*(jblock+2)));
                } else {
                    map(window*(iblock-1),cast(size_t)(window*(jblock-iblock+3)));
                }
            }
        }
    }

private:
    string filename;
    void[] data;
    ulong  start;
    size_t window;
    ulong  size;
    Mode   mMode;
    void*  address;

    version (Windows)
    {
        HANDLE hFile = INVALID_HANDLE_VALUE;
        HANDLE hFileMap = null;
        uint dwDesiredAccess;
    }
    else version (Posix)
    {
        int fd;
        int prot;
        int flags;
        int fmode;
    }
    else
    {
        static assert(0);
    }

    // Report error, where errno gives the error number
    // void errNo()
    // {
    //     version (Windows)
    //     {
    //         throw new FileException(filename, GetLastError());
    //     }
    //     else version (linux)
    //     {
    //         throw new FileException(filename, errno);
    //     }
    //     else
    //     {
    //         static assert(0);
    //     }
    // }
}

unittest
{
    import std.file : deleteme;

    const size_t K = 1024;
    size_t win = 64*K; // assume the page size is 64K
    version(Windows) {
        /+ these aren't defined in std.c.windows.windows so let's use default
         SYSTEM_INFO sysinfo;
         GetSystemInfo(&sysinfo);
         win = sysinfo.dwAllocationGranularity;
         +/
    } else version (linux) {
        // getpagesize() is not defined in the unix D headers so use the guess
    }
    string test_file = std.file.deleteme ~ "-testing.txt";
    MmFile mf = new MmFile(test_file,MmFile.Mode.readWriteNew,
            100*K,null,win);
    ubyte[] str = cast(ubyte[])"1234567890";
    ubyte[] data = cast(ubyte[])mf[0 .. 10];
    data[] = str[];
    assert( mf[0 .. 10] == str );
    data = cast(ubyte[])mf[50 .. 60];
    data[] = str[];
    assert( mf[50 .. 60] == str );
    ubyte[] data2 = cast(ubyte[])mf[20*K .. 60*K];
    assert( data2.length == 40*K );
    assert( data2[$-1] == 0 );
    mf[100*K-1] = cast(ubyte)'b';
    data2 = cast(ubyte[])mf[21*K .. 100*K];
    assert( data2.length == 79*K );
    assert( data2[$-1] == 'b' );
    delete mf;
    std.file.remove(test_file);
    // Create anonymous mapping
    auto test = new MmFile(null, MmFile.Mode.readWriteNew, 1024*1024, null);
}