// Written in the D programming language.

/* Written by Walter Bright and Andrei Alexandrescu
 * www.digitalmars.com
 * Placed in the Public Domain.
 */

/********************************
 * Standard I/O functions that extend $(B std.c.stdio).
 * $(B std.c.stdio) is $(D_PARAM public)ally imported when importing
 * $(B std.stdio).
 * Macros:
 *	WIKI=Phobos/StdStdio
 */

module std.stdio;

public import std.c.stdio;

import std.format;
import std.utf;
import std.string;
import std.gc;
import std.c.stdlib;
import std.c.string;
import std.c.stddef;
import std.conv;
import std.traits;
import std.contracts;
import std.file, std.typetuple; // for testing only

version (DigitalMars)
{
    version (Windows)
    {
	// Specific to the way Digital Mars C does stdio
	version = DIGITAL_MARS_STDIO;
    }
}

version (DIGITAL_MARS_STDIO)
{
}
else
{
    // Specific to the way Gnu C does stdio
    version = GCC_IO;
    import std.c.linux.linux;
}

version (DIGITAL_MARS_STDIO)
{
    extern (C)
    {
	/* **
	 * Digital Mars under-the-hood C I/O functions
	 */
	int _fputc_nlock(int, FILE*);
	int _fputwc_nlock(int, FILE*);
	int _fgetc_nlock(FILE*);
	int _fgetwc_nlock(FILE*);
	int __fp_lock(FILE*);
	void __fp_unlock(FILE*);
    }
    alias _fputc_nlock FPUTC;
    alias _fputwc_nlock FPUTWC;
    alias _fgetc_nlock FGETC;
    alias _fgetwc_nlock FGETWC;

    alias __fp_lock FLOCK;
    alias __fp_unlock FUNLOCK;
}
else version (GCC_IO)
     {
         /* **
          * Gnu under-the-hood C I/O functions; see
          * http://www.gnu.org/software/libc/manual/html_node/I_002fO-on-Streams.html#I_002fO-on-Streams
          */
         extern (C)
         {
             int fputc_unlocked(int, FILE*);
             int fputwc_unlocked(wchar_t, FILE*);
             int fgetc_unlocked(FILE*);
             int fgetwc_unlocked(FILE*);
             void flockfile(FILE*);
             void funlockfile(FILE*);
             ssize_t getline(char**, size_t*, FILE*);
             ssize_t getdelim (char**, size_t*, int, FILE*);

             private size_t fwrite_unlocked(const(void)* ptr,
                 size_t size, size_t n, FILE *stream);
         }

         alias fputc_unlocked FPUTC;
         alias fputwc_unlocked FPUTWC;
         alias fgetc_unlocked FGETC;
         alias fgetwc_unlocked FGETWC;

         alias flockfile FLOCK;
         alias funlockfile FUNLOCK;
     }
else
{
    static assert(0, "unsupported C I/O system");
}


/*********************
 * Thrown if I/O errors happen.
 */
class StdioException : Exception
{
    uint errno;			// operating system error code

    this(string msg)
    {
	super(msg);
    }

    this(uint errno)
    {	const(char*) s = std.string.strerror(errno);
	super(std.string.toString(s).idup);
    }

    static void opCall(string msg)
    {
	throw new StdioException(msg);
    }

    static void opCall()
    {
	throw new StdioException(getErrno());
    }
}

private
void writefx(FILE* fp, TypeInfo[] arguments, void* argptr, int newline=false)
{
    int orientation = fwide(fp, 0);

    /* Do the file stream locking at the outermost level
     * rather than character by character.
     */
    FLOCK(fp);
    scope(exit) FUNLOCK(fp);

    if (orientation <= 0)		// byte orientation or no orientation
    {
	void putc(dchar c)
	{
	    if (c <= 0x7F)
	    {
		FPUTC(c, fp);
	    }
	    else
	    {   char[4] buf;
		auto b = std.utf.toUTF8(buf, c);
		for (size_t i = 0; i < b.length; i++)
		    FPUTC(b[i], fp);
	    }
	}

	std.format.doFormat(&putc, arguments, argptr);
	if (newline)
	    FPUTC('\n', fp);
    }
    else if (orientation > 0)		// wide orientation
    {
	version (Windows)
	{
	    void putcw(dchar c)
	    {
		assert(isValidDchar(c));
		if (c <= 0xFFFF)
		{
		    FPUTWC(c, fp);
		}
		else
		{   wchar[2] buf;

		    buf[0] = cast(wchar) ((((c - 0x10000) >> 10) & 0x3FF) + 0xD800);
		    buf[1] = cast(wchar) (((c - 0x10000) & 0x3FF) + 0xDC00);
		    FPUTWC(buf[0], fp);
		    FPUTWC(buf[1], fp);
		}
	    }
	}
	else version (linux)
             {
                 void putcw(dchar c)
                 {
                     FPUTWC(c, fp);
                 }
             }
	else
	{
	    static assert(0);
	}

	std.format.doFormat(&putcw, arguments, argptr);
	if (newline)
	    FPUTWC('\n', fp);
    }
}


/***********************************
 * If the first argument $(D_PARAM args[0]) is a $(D_PARAM FILE*), for
 * each argument $(D_PARAM arg) in $(D_PARAM args[1..$]), format the
 * argument (as per $(LINK2 std_conv.html, to!(string)(arg))) and
 * write the resulting string to $(D_PARAM args[0]). If $(D_PARAM
 * args[0]) is not a $(D_PARAM FILE*), the call is equivalent to
 * $(D_PARAM write(stdout, args)).
 *
 * A call without any arguments will fail to compile. In the
 * exceedingly rare case you'd want to print a $(D_PARAM FILE*) to
 * $(D_PARAM stdout) as a hex pointer, $(D_PARAM write("", myFilePtr))
 * will do the trick.
 *
 * In case of an I/O error, throws an StdioException.
 */
void write(T...)(T args)
{
    // duplicate code for speed; no passing data around
    static if (!is(typeof(args[0]) : FILE*))
    {
        write(stdout, args);
    }
    else
    {
        FLOCK(args[0]);
        scope(exit) FUNLOCK(args[0]);
        foreach (arg; args[1 .. $])
        {
            final s = to!(string)(arg);
            size_t written = void;
            version(linux)
            {
                written = fwrite_unlocked(s.ptr,
                    s[0].sizeof, s.length, args[0]);
            }
            else
            {
                // TODO: figure out unlocked bulk writes on Windows
                written = std.c.stdio.fwrite(s.ptr,
                    s[0].sizeof, s.length, args[0]);
            }
            if (written != s.length)
            {
                StdioException();
            }
        }
    }
}

unittest
{
    // test write
    string file = "dmd-build-test.deleteme.txt";
    FILE* f = fopen(file, "w");
    assert(f);
    write(f, "Hello, ",  "world number ", 42, "!");
    fclose(f) == 0 || assert(false);
    assert(cast(char[]) std.file.read(file) == "Hello, world number 42!");
    // test write on stdout
    auto saveStdout = stdout;
    scope(exit) stdout = saveStdout;
    stdout = fopen(file, "w");
    assert(stdout);
    Object obj;
    write("Hello, ",  "world number ", 42, "! ", obj);
    fclose(stdout) == 0 || assert(false);
    auto result = cast(char[]) std.file.read(file);
    assert(result == "Hello, world number 42! null", result);
}

/***********************************
 * Equivalent to $(D_PARAM write(args, '\n')).  Calling $(D_PARAM
 * writeln) without arguments is valid and just prints a newline to
 * the standard output.
 */
void writeln(T...)(T args)
{
    write(args, '\n');
}

unittest
{
    // test writeln
    string file = "dmd-build-test.deleteme.txt";
    FILE* f = fopen(file, "w");
    assert(f);
    writeln(f, "Hello, ",  "world number ", 42, "!");
    fclose(f) == 0 || assert(false);
    assert(cast(char[]) std.file.read(file) == "Hello, world number 42!\n");
    // test writeln on stdout
    auto saveStdout = stdout;
    scope(exit) stdout = saveStdout;
    stdout = fopen(file, "w");
    assert(stdout);
    writeln("Hello, ",  "world number ", 42, "!");
    fclose(stdout) == 0 || assert(false);
    assert(cast(char[]) std.file.read(file) == "Hello, world number 42!\n");
}

/***********************************
 * If the first argument $(D_PARAM args[0]) is a $(D_PARAM FILE*), use
 * $(LINK2 std_format.html#format-string, the format specifier) in
 * $(D_PARAM args[1]) to control the formatting of $(D_PARAM
 * args[2..$]), and write the resulting string to $(D_PARAM args[0]).
 * If $(D_PARAM arg[0]) is not a $(D_PARAM FILE*), the call is
 * equivalent to $(D_PARAM writef(stdout, args)).
 *

 Warning:

 New behavior starting with D 2.006: unlike previous versions,
 $(D_PARAM writef) (and also $(D_PARAM writefln)) only scans its first
 string argument for format specifiers, but not subsequent string
 arguments. Also new in 2.006 is support for positional parameters
 with $(LINK2 http://www.opengroup.org/onlinepubs/009695399/functions/printf.html,POSIX)
 syntax.

Example:

-------------------------
writef("Date: %2$s %1$s", "October", 5); // "Date: 5 October"
------------------------

The positional and non-positional styles can be mixed in the same
format string. (POSIX leaves this behavior undefined.) The internal
counter for non-positional parameters tracks the next parameter after
the largest positional parameter already used. */

void writef(T...)(T args)
{
    FileWriter!(char) w;
    static const errorMessage =
        "You must pass a formatting string as the first"
        " argument to writef. If no formatting is needed,"
        " you may want to use write.";
    static if (!is(typeof(args[0]) : FILE*))
    {
        static if (!isSomeString!(T[0]))
        {
            // compatibility hack
            return writef(stdout, "", args);
        }
        else
        {
            w.backend = stdout;
            FLOCK(w.backend);
            scope(exit) FUNLOCK(w.backend);
            std.format.formattedWrite(w, args);
        }
    }
    else
    {
        static if (!isSomeString!(T[1]))
        {
            // compatibility hack
            return writef(args[0], "", args[1 .. $]);
        }
        else
        {
            w.backend = args[0];
            FLOCK(w.backend);
            scope(exit) FUNLOCK(w.backend);
            std.format.formattedWrite(w, args[1 .. $]);
        }
    }
}

unittest
{
    // test writef
    string file = "dmd-build-test.deleteme.txt";
    auto f = fopen(file, "w");
    assert(f);
    writef(f, "Hello, %s world number %s!", "nice", 42);
    fclose(f) == 0 || assert(false);
    assert(cast(char[]) std.file.read(file) ==  "Hello, nice world number 42!");
    // test write on stdout
    auto saveStdout = stdout;
    scope(exit) stdout = saveStdout;
    stdout = fopen(file, "w");
    assert(stdout);
    writef("Hello, %s world number %s!", "nice", 42);
    fclose(stdout) == 0 || assert(false);
    assert(cast(char[]) std.file.read(file) == "Hello, nice world number 42!");
}

/***********************************
 * Equivalent to $(D_PARAM writef(args, '\n')).
 */
void writefln(T...)(T args)
{
    //writef(args, '\n');
    // Duplicate code so we don't duplicate the stack; replace with macro l8r
    FileWriter!(char) w;
    static const errorMessage =
        "You must pass a formatting string as the first"
        " argument to writefln. If no formatting is needed,"
        " you may want to use writeln.";
    static if (!is(typeof(args[0]) : FILE*))
    {
        static if (!isSomeString!(T[0]))
        {
            // compatibility hack
            return writef(stdout, "", args, '\n');
        }
        else
        {
            w.backend = stdout;
            FLOCK(w.backend);
            scope(exit) FUNLOCK(w.backend);
            std.format.formattedWrite(w, args, '\n');
        }
    }
    else
    {
        static if (!isSomeString!(T[1]))
        {
            // compatibility hack
            return writef(args[0], "", args[1 .. $], '\n');
        }
        else
        {
            w.backend = args[0];
            FLOCK(w.backend);
            scope(exit) FUNLOCK(w.backend);
            std.format.formattedWrite(w, args[1 .. $], '\n');
        }
    }
}

unittest
{
    // test writefln
    string file = "dmd-build-test.deleteme.txt";
    FILE* f = fopen(file, "w");
    assert(f);
    writefln(f, "Hello, %s world number %s!", "nice", 42);
    fclose(f) == 0 || assert(false);
    assert(cast(char[]) std.file.read(file) == "Hello, nice world number 42!\n");
    // test write on stdout
    auto saveStdout = stdout;
    scope(exit) stdout = saveStdout;
    stdout = fopen(file, "w");
    assert(stdout);
    writefln("Hello, %s world number %s!", "nice", 42);
    fclose(stdout) == 0 || assert(false);
    assert(cast(char[]) std.file.read(file) == "Hello, nice world number 42!\n");
}

/***********************************
 * Kept for backward compatibility. Use $(D_PARAM writef) instead.
 */
void fwritef(FILE* fp, ...)
{
    writefx(fp, _arguments, _argptr, 0);
}

/***********************************
 * Kept for backward compatibility. Use $(D_PARAM writefln) instead.
 */
void fwritefln(FILE* fp, ...)
{
    writefx(fp, _arguments, _argptr, 1);    
}

/**********************************
 * Read line from stream $(D_PARAM fp).
 * Returns:
 *	$(D_PARAM null) for end of file,
 *	$(D_PARAM char[]) for line read from $(D_PARAM fp), including terminating character
 * Params:
 *	$(D_PARAM fp) = input stream
 *	$(D_PARAM terminator) = line terminator, '\n' by default
 * Throws:
 *	$(D_PARAM StdioException) on error
 * Example:
 *	Reads $(D_PARAM stdin) and writes it to $(D_PARAM stdout).
---
import std.stdio;

int main()
{
    char[] buf;
    while ((buf = readln()) != null)
	write(buf);
    return 0;
}
---
*/
string readln(FILE* fp = stdin, dchar terminator = '\n')
{
    char[] buf;
    readln(fp, buf, terminator);
    return assumeUnique(buf);
}

/**********************************
 * Read line from stream $(D_PARAM fp) and write it to $(D_PARAM
 * buf[]), including terminating character.
 *
 * This is often faster than $(D_PARAM readln(FILE*)) because the buffer
 * is reused each call. Note that reusing the buffer means that
 * the previous contents of it need to be copied if needed.
 * Params:
 *	$(D_PARAM fp) = input stream
 *	$(D_PARAM buf) = buffer used to store the resulting line data. buf
 *		is resized as necessary.
 * Returns:
 *	0 for end of file, otherwise
 *	number of characters read
 * Throws:
 *	$(D_PARAM StdioException) on error
 * Example:
 *	Reads $(D_PARAM stdin) and writes it to $(D_PARAM stdout).
---
import std.stdio;

int main()
{
    char[] buf;
    while (readln(stdin, buf))
	write(buf);
    return 0;
}
---
This method is more efficient than the one in the previous example
because $(D_PARAM readln(stdin, buf)) reuses (if possible) memory
allocated by $(D_PARAM buf), whereas $(D_PARAM buf = readln()) makes a
new memory allocation with every line.
*/
size_t readln(FILE* fp, inout char[] buf, dchar terminator = '\n')
{
    version (DIGITAL_MARS_STDIO)
    {
	FLOCK(fp);
	scope(exit) FUNLOCK(fp);

	if (__fhnd_info[fp._file] & FHND_WCHAR)
	{   /* Stream is in wide characters.
	     * Read them and convert to chars.
	     */
	    static assert(wchar_t.sizeof == 2);
	    buf.length = 0;
	    int c2;
	    for (int c; (c = FGETWC(fp)) != -1; )
	    {
		if ((c & ~0x7F) == 0)
		{   buf ~= c;
		    if (c == terminator)
			break;
		}
		else
		{
		    if (c >= 0xD800 && c <= 0xDBFF)
		    {
			if ((c2 = FGETWC(fp)) != -1 ||
			    c2 < 0xDC00 && c2 > 0xDFFF)
			{
			    StdioException("unpaired UTF-16 surrogate");
			}
			c = ((c - 0xD7C0) << 10) + (c2 - 0xDC00);
		    }
		    std.utf.encode(buf, c);
		}
	    }
	    if (ferror(fp))
		StdioException();
	    return buf.length;
	}

	auto sz = std.gc.capacity(buf.ptr);
	//auto sz = buf.length;
	buf = buf.ptr[0 .. sz];
	if (fp._flag & _IONBF)
	{
	    /* Use this for unbuffered I/O, when running
	     * across buffer boundaries, or for any but the common
	     * cases.
	     */
        L1:
	    char *p;

	    if (sz)
	    {
		p = buf.ptr;
	    }
	    else
	    {
		sz = 64;
		p = cast(char*) std.gc.malloc(sz);
		std.gc.hasNoPointers(p);
		buf = p[0 .. sz];
	    }
	    size_t i = 0;
	    for (int c; (c = FGETC(fp)) != -1; )
	    {
		if ((p[i] = c) != terminator)
		{
		    i++;
		    if (i < sz)
			continue;
		    buf = p[0 .. i] ~ readln(fp);
		    return buf.length;
		}
		else
		{
		    buf = p[0 .. i + 1];
		    return i + 1;
		}
	    }
	    if (ferror(fp))
		StdioException();
	    buf = p[0 .. i];
	    return i;
	}
	else
	{
	    int u = fp._cnt;
	    char* p = fp._ptr;
	    int i;
	    if (fp._flag & _IOTRAN)
	    {   /* Translated mode ignores \r and treats ^Z as end-of-file
		 */
		char c;
		while (1)
		{
		    if (i == u)		// if end of buffer
			goto L1;	// give up
		    c = p[i];
		    i++;
		    if (c != '\r')
		    {
			if (c == terminator)
			    break;
			if (c != 0x1A)
			    continue;
			goto L1;
		    }
		    else
		    {   if (i != u && p[i] == terminator)
			    break;
			goto L1;
		    }
		}
		if (i > sz)
		{
		    buf = cast(char[])std.gc.malloc(i);
		    std.gc.hasNoPointers(buf.ptr);
		}
		if (i - 1)
		    memcpy(buf.ptr, p, i - 1);
		buf[i - 1] = terminator;
		if (terminator == '\n' && c == '\r')
		    i++;
	    }
	    else
	    {
		while (1)
		{
		    if (i == u)		// if end of buffer
			goto L1;	// give up
		    auto c = p[i];
		    i++;
		    if (c == terminator)
			break;
		}
		if (i > sz)
		{
		    buf = cast(char[])std.gc.malloc(i);
		    std.gc.hasNoPointers(buf.ptr);
		}
		memcpy(buf.ptr, p, i);
	    }
	    fp._cnt -= i;
	    fp._ptr += i;
	    buf = buf[0 .. i];
	    return i;
	}
    }
    else version (GCC_IO)
         {
             if (fwide(fp, 0) > 0)
             {   /* Stream is in wide characters.
                  * Read them and convert to chars.
                  */
                 FLOCK(fp);
                 scope(exit) FUNLOCK(fp);
                 version (Windows)
                 {
                     buf.length = 0;
                     int c2;
                     for (int c; (c = FGETWC(fp)) != -1; )
                     {
                         if ((c & ~0x7F) == 0)
                         {   buf ~= c;
                             if (c == terminator)
                                 break;
                         }
                         else
                         {
                             if (c >= 0xD800 && c <= 0xDBFF)
                             {
                                 if ((c2 = FGETWC(fp)) != -1 ||
                                     c2 < 0xDC00 && c2 > 0xDFFF)
                                 {
                                     StdioException("unpaired UTF-16 surrogate");
                                 }
                                 c = ((c - 0xD7C0) << 10) + (c2 - 0xDC00);
                             }
                             std.utf.encode(buf, c);
                         }
                     }
                     if (ferror(fp))
                         StdioException();
                     return buf.length;
                 }
                 else version (linux)
                      {
                          buf.length = 0;
                          for (int c; (c = FGETWC(fp)) != -1; )
                          {
                              if ((c & ~0x7F) == 0)
                                  buf ~= c;
                              else
                                  std.utf.encode(buf, cast(dchar)c);
                              if (c == terminator)
                                  break;
                          }
                          if (ferror(fp))
                              StdioException();
                          return buf.length;
                      }
                 else
                 {
                     static assert(0);
                 }
             }

             char *lineptr = null;
             size_t n = 0;
             auto s = getdelim(&lineptr, &n, terminator, fp);
             scope(exit) free(lineptr);
             if (s < 0)
             {
                 if (ferror(fp))
                     StdioException();
                 buf.length = 0;		// end of file
                 return 0;
             }
             buf = buf.ptr[0 .. std.gc.capacity(buf.ptr)];
             if (s <= buf.length)
             {
                 buf.length = s;
                 buf[] = lineptr[0 .. s];
             }
             else
             {
                 buf = lineptr[0 .. s].dup;
             }
             return s;
         }
    else
    {
	static assert(0);
    }
}

/** ditto */
size_t readln(inout char[] buf, dchar terminator = '\n')
{
    return readln(stdin, buf, terminator);
}

/***********************************
 * Convenience function that forwards to $(D_PARAM std.c.stdio.fopen)
 * with appropriately-constructed C-style strings.
 */
FILE* fopen(string name, string mode = "r")
{
    return std.c.stdio.fopen(toStringz(name), toStringz(mode));
}

extern(C) FILE* popen(const char*, const char*);

/***********************************
 * Convenience function that forwards to $(D_PARAM std.c.stdio.popen)
 * with appropriately-constructed C-style strings.
 */
FILE* popen(string name, string mode)
{
    return popen(toStringz(name), toStringz(mode));
}

/*
 * Implements the static Writer interface for a FILE*. Instantiate it
 * with the character type, e.g. FileWriter!(char),
 * FileWriter!(wchar), or FileWriter!(dchar). Regardless of
 * instantiation, FileWriter supports all character widths; it only is
 * the most efficient at accepting the character type it was
 * instantiated with.
 *
 * */
private struct FileWriter(Char)
{
    alias Char NativeChar;
    FILE* backend;
    int orientation;
    void write(C)(in C[] s)
    {
        if (!orientation) orientation = fwide(backend, 0);
        if (orientation <= 0 && C.sizeof == 1)
        {
            // lucky case: narrow chars on narrow stream
            if (std.c.stdio.fwrite(s.ptr, C.sizeof, s.length, backend)
                != s.length)
            {
                StdioException();
            }
        }
        else
        {
            // put each character in turn
            foreach (C c; s)
            {
                putchar(c);
            }
        }
    }
    void putchar(C)(in C c)
    {
        if (!orientation) orientation = fwide(backend, 0);
        static if (c.sizeof == 1)
        {
            // simple char
            if (orientation <= 0) FPUTC(c, backend);
            else FPUTWC(c, backend);
        }
        else static if (c.sizeof == 2)
        {
            if (orientation <= 0)
            {
                if (c <= 0x7F)
                {
                    FPUTC(c, backend);
                }
                else
                {
                    char[4] buf;
                    auto b = std.utf.toUTF8(buf, c);
                    foreach (i ; 0 .. b.length)
                        FPUTC(b[i], backend);
                }
            }
            else
            {
                FPUTWC(c, backend);
            }
        }
        else // 32-bit characters
        {
            if (orientation <= 0)
            {
                if (c <= 0x7F)
                {
                    FPUTC(c, backend);
                }
                else
                {
                    char[4] buf;
                    auto b = std.utf.toUTF8(buf, c);
                    foreach (i ; 0 .. b.length)
                        FPUTC(b[i], backend);
                }
            }
            else
            {
                version (Windows)
                {
                    assert(isValidDchar(c));
                    if (c <= 0xFFFF)
                    {
                        FPUTWC(c, backend);
                    }
                    else
                    {
                        FPUTWC(cast(wchar) ((((c - 0x10000) >> 10) & 0x3FF)
                                            + 0xD800), backend);
                        FPUTWC(cast(wchar) (((c - 0x10000) & 0x3FF) + 0xDC00),
                               backend);
                    }
                }
                else version (linux)
                {
                    FPUTWC(c, backend);
                }
                else
                {
                    static assert(0);
                }
            }
        }
    }
}

struct LinesIterator
{
    FILE * f;
    dchar terminator;
    // new S allocated every time
    private int opApplyString(S, D)(D dg)
    {
        alias ParameterTypeTuple!(dg) Parms;
        int result = 1;
        string line;
        ulong i;
        while ((line = readln(f, terminator)).length)
        {
            auto copy = to!(Parms[$ - 1])(line);
            static if (Parms.length == 2)
                result = dg(i, copy);
            else
                result = dg(copy);
            if (result != 0) break;
            static if (Parms.length == 2) ++line;
        }
        return result;
    }
    // new string allocated every time
    int opApply(int delegate(ref string) dg)
    {
        return opApplyString!(string, typeof(dg))(dg);
    }
    // new wstring allocated every time
    int opApply(int delegate(ref wstring) dg)
    {
        return opApplyString!(wstring, typeof(dg))(dg);
    }
    // new dstring allocated every time
    int opApply(int delegate(ref dstring) dg)
    {
        return opApplyString!(dstring, typeof(dg))(dg);
    }

    // string is reused between calls
    int opApply(int delegate(ref char[]) dg)
    {
        int result = 1;
        char[] line;
        while (readln(f, line, terminator))
        {
            if ((result = dg(line)) != 0) break;
        }
        return result;
    }
    // w/dstring is reused between calls
    // TODO: optimize this
    private int opApplyReuseWideString(Char)(int delegate(ref Char[]) dg)
    {
        int result = 1;
        char[] line;
        Char[] copy;
        while (readln(f, line, terminator))
        {
            foreach (size_t i, Char c; line)
            {
                if (i >= copy.length) copy ~= c;
                else copy[i] = c;
            }
            if ((result = dg(copy)) != 0) break;
        }
        return result;
    }
    int opApply(int delegate(ref wchar[]) dg)
    {
        return opApplyReuseWideString(dg);
    }
    int opApply(int delegate(ref dchar[]) dg)
    {
        return opApplyReuseWideString(dg);
    }
    // no UTF checking, new allocation every line
    int opApply(int delegate(ref invariant(ubyte)[]) dg)
    {
        int result = 1;
        int c = void;
        FLOCK(f);
	scope(exit) FUNLOCK(f);
        ubyte[] buffer;
        while ((c = FGETC(f)) != -1)
        {
            buffer ~= to!(ubyte)(c);
            if (c == terminator)
            {
                auto fake = assumeUnique(buffer);
                // unlock the file while calling the delegate
                FUNLOCK(f);
                scope(exit) FLOCK(f);
                if ((result = dg(fake)) != 0) break;
            }
        }
        return result;
    }
    // no UTF checking, reuse buffer
    int opApply(int delegate(ref ubyte[]) dg)
    {
        return opApplyGeneric(dg);
    }
    int opApply(int delegate(ref ulong i, ref ubyte[]) dg)
    {
        return opApplyGeneric(dg);
    }
    
    int opApplyGeneric(D)(D dg)
    {
        alias ParameterTypeTuple!(dg) Parms;
        int result = 1;
        int c = void;
        FLOCK(f);
	scope(exit) FUNLOCK(f);
        Parms[$ - 1] buffer;
        static if (Parms.length == 2) Parms[0] i;
        while ((c = FGETC(f)) != -1)
        {
            buffer ~= to!(typeof(buffer[0]))(c);
            if (c == terminator)
            {
                // unlock the file while calling the delegate
                FUNLOCK(f);
                scope(exit) FLOCK(f);
                static if (Parms.length == 2)
                    result = dg(i, buffer);
                else
                    result = dg(buffer);
                if (result != 0) return result;
                buffer.length = 0;
                static if (Parms.length == 2) ++i;
            }
        }
        // can only reach when FGETC returned -1
        if (!feof(f)) throw new StdioException(ferror(f)); // error occured
        return result;
    }
}

/**
 * Iterates through the lines of a file by using $(D_PARAM foreach).
 *
 * Example:
 *
---------
void main()
{
  foreach (string line; lines(stdin))
  {
    ... use line ...
  }
}
---------
 The line terminator ('\n' by default) is part of the string read (it
could be missing in the last line of the file). Several types are
supported for $(D_PARAM line), and the behavior of $(D_PARAM lines)
changes accordingly:

$(OL $(LI If $(D_PARAM line) has type $(D_PARAM string), $(D_PARAM
wstring), or $(D_PARAM dstring), a new string of the respective type
is allocated every read.) $(LI If $(D_PARAM line) has type $(D_PARAM
char[]), $(D_PARAM wchar[]), $(D_PARAM dchar[]), the line's content
will be reused (overwritten) across reads.) $(LI If $(D_PARAM line)
has type $(D_PARAM invariant(ubyte)[]), the behavior is similar to
case (1), except that no UTF checking is attempted upon input.) $(LI
If $(D_PARAM line) has type $(D_PARAM ubyte[]), the behavior is
similar to case (2), except that no UTF checking is attempted upon
input.))

In all cases, a two-symbols versions is also accepted, in which case
the first symbol (of type $(D_PARAM ulong)) tracks the zero-based
number of the current line.

Example:
----
  foreach (ulong i, string line; lines(stdin))
  {
    ... use line ...
  }
----

 In case of an I/O error, an $(D_PARAM StdioException) is thrown. 
 */

LinesIterator lines(FILE* f, dchar terminator = '\n')
{
    LinesIterator result;
    result.f = f;
    result.terminator = terminator;
    return result;
}

// / ditto
// LinesIterator lines(dchar terminator = '\n')
// {
//     return lines(stdin, terminator);
// }

unittest
{
    string file = "dmd-build-test.deleteme.txt";
    alias TypeTuple!(string, wstring, dstring,
                     char[], wchar[], dchar[])
        TestedWith;
    foreach (T; TestedWith) {
        // test looping with an empty file
        std.file.write(file, "");
        auto f = fopen(file, "r");
        foreach (T line; lines(f))
        {
            assert(false);
        }
        fclose(f) == 0 || assert(false);

        // test looping with a file with three lines
        std.file.write(file, "Line one\nline two\nline three\n");
        f = fopen(file, "r");
        uint i = 0;
        foreach (T line; lines(f))
        {
            if (i == 0) assert(line == "Line one\n");
            else if (i == 1) assert(line == "line two\n");
            else if (i == 2) assert(line == "line three\n");
            else assert(false);
            ++i;
        }
        fclose(f) == 0 || assert(false);
    
        // test looping with a file with three lines, last without a newline
        std.file.write(file, "Line one\nline two\nline three");
        f = fopen(file, "r");
        i = 0;
        foreach (T line; lines(f))
        {
            if (i == 0) assert(line == "Line one\n");
            else if (i == 1) assert(line == "line two\n");
            else if (i == 2) assert(line == "line three");
            else assert(false);
            ++i;
        }
        fclose(f) == 0 || assert(false);
    }

    // test with ubyte[] inputs
    alias TypeTuple!(invariant(ubyte)[], ubyte[])
        TestedWith2;
    foreach (T; TestedWith2) {
        // test looping with an empty file
        std.file.write(file, "");
        auto f = fopen(file, "r");
        foreach (T line; lines(f))
        {
            assert(false);
        }
        fclose(f) == 0 || assert(false);

        // test looping with a file with three lines
        std.file.write(file, "Line one\nline two\nline three\n");
        f = fopen(file, "r");
        uint i = 0;
        foreach (T line; lines(f))
        {
            if (i == 0) assert(cast(char[]) line == "Line one\n");
            else if (i == 1) assert(cast(char[]) line == "line two\n");
            else if (i == 2) assert(cast(char[]) line == "line three\n");
            else assert(false);
            ++i;
        }
        fclose(f) == 0 || assert(false);
    
        // test looping with a file with three lines, last without a newline
        std.file.write(file, "Line one\nline two\nline three");
        f = fopen(file, "r");
        i = 0;
        foreach (T line; lines(f))
        {
            if (i == 0) assert(cast(char[]) line == "Line one\n");
            else if (i == 1) assert(cast(char[]) line == "line two\n");
            else if (i == 2) assert(cast(char[]) line == "line three");
            else assert(false);
            ++i;
        }
        fclose(f) == 0 || assert(false);

    }

    foreach (T; TypeTuple!(ubyte[]))
    {
        // test looping with a file with three lines, last without a newline
        // using a counter too this time
        std.file.write(file, "Line one\nline two\nline three");
        auto f = fopen(file, "r");
        uint i = 0;
        foreach (ulong j, T line; lines(f))
        {
            if (i == 0) assert(cast(char[]) line == "Line one\n");
            else if (i == 1) assert(cast(char[]) line == "line two\n");
            else if (i == 2) assert(cast(char[]) line == "line three");
            else assert(false);
            ++i;
        }
        fclose(f) == 0 || assert(false);
    }
}

struct ChunkIterator
{
    private FILE* f;
    private size_t size;
    int opApply(int delegate(ref ubyte[]) dg)
    {
        const maxStackSize = 500 * 1024;
        ubyte[] buffer = void;
        if (size < maxStackSize)
            buffer = (cast(ubyte*) alloca(size))[0 .. size];
        else
            buffer = new ubyte[size];
        size_t r = void;
        int result = 1;
        while ((r = std.c.stdio.fread(buffer.ptr,
                                      buffer[0].sizeof, size, f)) > 0)
        {
            assert(r <= size);
            if (r != size)
            {
                // error occured
                if (!feof(f)) throw new StdioException(ferror(f));
                if (!r) break; // done reading
            }
            if ((result = dg(buffer)) != 0) break;
        }
        return result;
    }
}

/**
Iterates through a file a chunk at a time by using $(D_PARAM
foreach).

Example:

---------
void main()
{
  foreach (ubyte[] buffer; chunks(stdin, 4096))
  {
    ... use buffer ...
  }
}
---------

The content of $(D_PARAM buffer) is reused across calls. In the
 example above, $(D_PARAM buffer.length) is 4096 for all iterations,
 except for the last one, in which case $(D_PARAM buffer.length) may
 be less than 4096 (but always greater than zero).

 In case of an I/O error, an $(D_PARAM StdioException) is thrown. 
*/

ChunkIterator chunks(FILE* f, size_t size)
{
    return ChunkIterator(f, size);
}

unittest
{
    string file = "dmd-build-test.deleteme.txt";
    // test looping with an empty file
    std.file.write(file, "");
    auto f = fopen(file, "r");
    foreach (ubyte[] line; chunks(f, 4))
    {
        assert(false);
    }
    fclose(f) == 0 || assert(false);
    
    // test looping with a file with three lines
    std.file.write(file, "Line one\nline two\nline three\n");
    f = fopen(file, "r");
    uint i = 0;
    foreach (ubyte[] line; ChunkIterator(f, 3))
    {
        if (i == 0) assert(cast(char[]) line == "Lin");
        else if (i == 1) assert(cast(char[]) line == "e o");
        else if (i == 2) assert(cast(char[]) line == "ne\n");
        else break;
        ++i;
    }
    fclose(f) == 0 || assert(false);
}