py/stream.c - lite/micropython - Linaro Git Browser

 /*
  * This file is part of the MicroPython project, http://micropython.org/
  *
  * The MIT License (MIT)
  *
  * Copyright (c) 2013, 2014 Damien P. George
  * Copyright (c) 2014 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
  * in the Software without restriction, including without limitation the rights
  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  * copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in
  * all copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */

 #include <string.h>
 #include <unistd.h>

 #include "py/objstr.h"
 #include "py/stream.h"
 #include "py/runtime.h"

 #if MICROPY_STREAMS_NON_BLOCK
 #include <errno.h>
 #if defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR)
 #define EWOULDBLOCK 140
 #endif
 #endif

 // This file defines generic Python stream read/write methods which
 // dispatch to the underlying stream interface of an object.

 // TODO: should be in mpconfig.h
 #define DEFAULT_BUFFER_SIZE 256

 STATIC mp_obj_t stream_readall(mp_obj_t self_in);

 #define STREAM_CONTENT_TYPE(stream) (((stream)->is_text) ? &mp_type_str : &mp_type_bytes)

 // Returns error condition in *errcode, if non-zero, return value is number of bytes written
 // before error condition occurred. If *errcode == 0, returns total bytes written (which will
 // be equal to input size).
 mp_uint_t mp_stream_rw(mp_obj_t stream, void *buf_, mp_uint_t size, int *errcode, byte flags) {
     byte *buf = buf_;
     mp_obj_base_t* s = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
     typedef mp_uint_t (*io_func_t)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode);
     io_func_t io_func;
     const mp_stream_p_t *stream_p = s->type->protocol;
     if (flags & MP_STREAM_RW_WRITE) {
         io_func = (io_func_t)stream_p->write;
     } else {
         io_func = stream_p->read;
     }

     *errcode = 0;
     mp_uint_t done = 0;
     while (size > 0) {
         mp_uint_t out_sz = io_func(stream, buf, size, errcode);
         // For read, out_sz == 0 means EOF. For write, it's unspecified
         // what it means, but we don't make any progress, so returning
         // is still the best option.
         if (out_sz == 0) {
             return done;
         }
         if (out_sz == MP_STREAM_ERROR) {
             // If we read something before getting EAGAIN, don't leak it
             if (mp_is_nonblocking_error(*errcode) && done != 0) {
                 *errcode = 0;
             }
             return done;
         }
         if (flags & MP_STREAM_RW_ONCE) {
             return out_sz;
         }

         buf += out_sz;
         size -= out_sz;
         done += out_sz;
     }
     return done;
 }

 const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags) {
     mp_obj_type_t *type = mp_obj_get_type(self_in);
     const mp_stream_p_t *stream_p = type->protocol;
     if (stream_p == NULL
         || ((flags & MP_STREAM_OP_READ) && stream_p->read == NULL)
         || ((flags & MP_STREAM_OP_WRITE) && stream_p->write == NULL)
         || ((flags & MP_STREAM_OP_IOCTL) && stream_p->ioctl == NULL)) {
         // CPython: io.UnsupportedOperation, OSError subclass
         mp_raise_msg(&mp_type_OSError, "stream operation not supported");
     }
     return stream_p;
 }

 mp_obj_t mp_stream_close(mp_obj_t stream) {
     // TODO: Still consider using ioctl for close
     mp_obj_t dest[2];
     mp_load_method(stream, MP_QSTR_close, dest);
     return mp_call_method_n_kw(0, 0, dest);
 }

 STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte flags) {
     const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ);

     // What to do if sz < -1?  Python docs don't specify this case.
     // CPython does a readall, but here we silently let negatives through,
     // and they will cause a MemoryError.
     mp_int_t sz;
     if (n_args == 1 || ((sz = mp_obj_get_int(args[1])) == -1)) {
         return stream_readall(args[0]);
     }

     #if MICROPY_PY_BUILTINS_STR_UNICODE
     if (stream_p->is_text) {
         // We need to read sz number of unicode characters.  Because we don't have any
         // buffering, and because the stream API can only read bytes, we must read here
         // in units of bytes and must never over read.  If we want sz chars, then reading
         // sz bytes will never over-read, so we follow this approach, in a loop to keep
         // reading until we have exactly enough chars.  This will be 1 read for text
         // with ASCII-only chars, and about 2 reads for text with a couple of non-ASCII
         // chars.  For text with lots of non-ASCII chars, it'll be pretty inefficient
         // in time and memory.

         vstr_t vstr;
         vstr_init(&vstr, sz);
         mp_uint_t more_bytes = sz;
         mp_uint_t last_buf_offset = 0;
         while (more_bytes > 0) {
             char *p = vstr_add_len(&vstr, more_bytes);
             int error;
             mp_uint_t out_sz = mp_stream_read_exactly(args[0], p, more_bytes, &error);
             if (error != 0) {
                 vstr_cut_tail_bytes(&vstr, more_bytes);
                 if (mp_is_nonblocking_error(error)) {
                     // With non-blocking streams, we read as much as we can.
                     // If we read nothing, return None, just like read().
                     // Otherwise, return data read so far.
                     // TODO what if we have read only half a non-ASCII char?
                     if (vstr.len == 0) {
                         vstr_clear(&vstr);
                         return mp_const_none;
                     }
                     break;
                 }
                 mp_raise_OSError(error);
             }

             if (out_sz < more_bytes) {
                 // Finish reading.
                 // TODO what if we have read only half a non-ASCII char?
                 vstr_cut_tail_bytes(&vstr, more_bytes - out_sz);
                 if (out_sz == 0) {
                     break;
                 }
             }

             // count chars from bytes just read
             for (mp_uint_t off = last_buf_offset;;) {
                 byte b = vstr.buf[off];
                 int n;
                 if (!UTF8_IS_NONASCII(b)) {
                     // 1-byte ASCII char
                     n = 1;
                 } else if ((b & 0xe0) == 0xc0) {
                     // 2-byte char
                     n = 2;
                 } else if ((b & 0xf0) == 0xe0) {
                     // 3-byte char
                     n = 3;
                 } else if ((b & 0xf8) == 0xf0) {
                     // 4-byte char
                     n = 4;
                 } else {
                     // TODO
                     n = 5;
                 }
                 if (off + n <= vstr.len) {
                     // got a whole char in n bytes
                     off += n;
                     sz -= 1;
                     last_buf_offset = off;
                     if (off >= vstr.len) {
                         more_bytes = sz;
                         break;
                     }
                 } else {
                     // didn't get a whole char, so work out how many extra bytes are needed for
                     // this partial char, plus bytes for additional chars that we want
                     more_bytes = (off + n - vstr.len) + (sz - 1);
                     break;
                 }
             }
         }

         return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);
     }
     #endif

     vstr_t vstr;
     vstr_init_len(&vstr, sz);
     int error;
     mp_uint_t out_sz = mp_stream_rw(args[0], vstr.buf, sz, &error, flags);
     if (error != 0) {
         vstr_clear(&vstr);
         if (mp_is_nonblocking_error(error)) {
             // https://docs.python.org/3.4/library/io.html#io.RawIOBase.read
             // "If the object is in non-blocking mode and no bytes are available,
             // None is returned."
             // This is actually very weird, as naive truth check will treat
             // this as EOF.
             return mp_const_none;
         }
         mp_raise_OSError(error);
     } else {
         vstr.len = out_sz;
         return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
     }
 }

 STATIC mp_obj_t stream_read(size_t n_args, const mp_obj_t *args) {
     return stream_read_generic(n_args, args, MP_STREAM_RW_READ);
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read_obj, 1, 2, stream_read);

 STATIC mp_obj_t stream_read1(size_t n_args, const mp_obj_t *args) {
     return stream_read_generic(n_args, args, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read1_obj, 1, 2, stream_read1);

 mp_obj_t mp_stream_write(mp_obj_t self_in, const void *buf, size_t len, byte flags) {
     mp_get_stream_raise(self_in, MP_STREAM_OP_WRITE);

     int error;
     mp_uint_t out_sz = mp_stream_rw(self_in, (void*)buf, len, &error, flags);
     if (error != 0) {
         if (mp_is_nonblocking_error(error)) {
             // http://docs.python.org/3/library/io.html#io.RawIOBase.write
             // "None is returned if the raw stream is set not to block and
             // no single byte could be readily written to it."
             return mp_const_none;
         }
         mp_raise_OSError(error);
     } else {
         return MP_OBJ_NEW_SMALL_INT(out_sz);
     }
 }

 // XXX hack
 void mp_stream_write_adaptor(void *self, const char *buf, size_t len) {
     mp_stream_write(MP_OBJ_FROM_PTR(self), buf, len, MP_STREAM_RW_WRITE);
 }

 STATIC mp_obj_t stream_write_method(size_t n_args, const mp_obj_t *args) {
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ);
     size_t max_len = (size_t)-1;
     size_t off = 0;
     if (n_args == 3) {
         max_len = mp_obj_get_int_truncated(args[2]);
     } else if (n_args == 4) {
         off = mp_obj_get_int_truncated(args[2]);
         max_len = mp_obj_get_int_truncated(args[3]);
         if (off > bufinfo.len) {
             off = bufinfo.len;
         }
     }
     bufinfo.len -= off;
     return mp_stream_write(args[0], (byte*)bufinfo.buf + off, MIN(bufinfo.len, max_len), MP_STREAM_RW_WRITE);
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj, 2, 4, stream_write_method);

 STATIC mp_obj_t stream_write1_method(mp_obj_t self_in, mp_obj_t arg) {
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ);
     return mp_stream_write(self_in, bufinfo.buf, bufinfo.len, MP_STREAM_RW_WRITE | MP_STREAM_RW_ONCE);
 }
 MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_write1_obj, stream_write1_method);

 STATIC mp_obj_t stream_readinto(size_t n_args, const mp_obj_t *args) {
     mp_get_stream_raise(args[0], MP_STREAM_OP_READ);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_WRITE);

     // CPython extension: if 2nd arg is provided, that's max len to read,
     // instead of full buffer. Similar to
     // https://docs.python.org/3/library/socket.html#socket.socket.recv_into
     mp_uint_t len = bufinfo.len;
     if (n_args > 2) {
         len = mp_obj_get_int(args[2]);
         if (len > bufinfo.len) {
             len = bufinfo.len;
         }
     }

     int error;
     mp_uint_t out_sz = mp_stream_read_exactly(args[0], bufinfo.buf, len, &error);
     if (error != 0) {
         if (mp_is_nonblocking_error(error)) {
             return mp_const_none;
         }
         mp_raise_OSError(error);
     } else {
         return MP_OBJ_NEW_SMALL_INT(out_sz);
     }
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto_obj, 2, 3, stream_readinto);

 STATIC mp_obj_t stream_readall(mp_obj_t self_in) {
     const mp_stream_p_t *stream_p = mp_get_stream_raise(self_in, MP_STREAM_OP_READ);

     mp_uint_t total_size = 0;
     vstr_t vstr;
     vstr_init(&vstr, DEFAULT_BUFFER_SIZE);
     char *p = vstr.buf;
     mp_uint_t current_read = DEFAULT_BUFFER_SIZE;
     while (true) {
         int error;
         mp_uint_t out_sz = stream_p->read(self_in, p, current_read, &error);
         if (out_sz == MP_STREAM_ERROR) {
             if (mp_is_nonblocking_error(error)) {
                 // With non-blocking streams, we read as much as we can.
                 // If we read nothing, return None, just like read().
                 // Otherwise, return data read so far.
                 if (total_size == 0) {
                     return mp_const_none;
                 }
                 break;
             }
             mp_raise_OSError(error);
         }
         if (out_sz == 0) {
             break;
         }
         total_size += out_sz;
         if (out_sz < current_read) {
             current_read -= out_sz;
             p += out_sz;
         } else {
             p = vstr_extend(&vstr, DEFAULT_BUFFER_SIZE);
             current_read = DEFAULT_BUFFER_SIZE;
         }
     }

     vstr.len = total_size;
     return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
 }

 // Unbuffered, inefficient implementation of readline() for raw I/O files.
 STATIC mp_obj_t stream_unbuffered_readline(size_t n_args, const mp_obj_t *args) {
     const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ);

     mp_int_t max_size = -1;
     if (n_args > 1) {
         max_size = MP_OBJ_SMALL_INT_VALUE(args[1]);
     }

     vstr_t vstr;
     if (max_size != -1) {
         vstr_init(&vstr, max_size);
     } else {
         vstr_init(&vstr, 16);
     }

     while (max_size == -1 || max_size-- != 0) {
         char *p = vstr_add_len(&vstr, 1);
         int error;
         mp_uint_t out_sz = stream_p->read(args[0], p, 1, &error);
         if (out_sz == MP_STREAM_ERROR) {
             if (mp_is_nonblocking_error(error)) {
                 if (vstr.len == 1) {
                     // We just incremented it, but otherwise we read nothing
                     // and immediately got EAGAIN. This case is not well
                     // specified in
                     // https://docs.python.org/3/library/io.html#io.IOBase.readline
                     // unlike similar case for read(). But we follow the latter's
                     // behavior - return None.
                     vstr_clear(&vstr);
                     return mp_const_none;
                 } else {
                     goto done;
                 }
             }
             mp_raise_OSError(error);
         }
         if (out_sz == 0) {
 done:
             // Back out previously added byte
             // Consider, what's better - read a char and get OutOfMemory (so read
             // char is lost), or allocate first as we do.
             vstr_cut_tail_bytes(&vstr, 1);
             break;
         }
         if (*p == '\n') {
             break;
         }
     }

     return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj, 1, 2, stream_unbuffered_readline);

 // TODO take an optional extra argument (what does it do exactly?)
 STATIC mp_obj_t stream_unbuffered_readlines(mp_obj_t self) {
     mp_obj_t lines = mp_obj_new_list(0, NULL);
     for (;;) {
         mp_obj_t line = stream_unbuffered_readline(1, &self);
         if (!mp_obj_is_true(line)) {
             break;
         }
         mp_obj_list_append(lines, line);
     }
     return lines;
 }
 MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj, stream_unbuffered_readlines);

 mp_obj_t mp_stream_unbuffered_iter(mp_obj_t self) {
     mp_obj_t l_in = stream_unbuffered_readline(1, &self);
     if (mp_obj_is_true(l_in)) {
         return l_in;
     }
     return MP_OBJ_STOP_ITERATION;
 }

 STATIC mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) {
     const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL);

     struct mp_stream_seek_t seek_s;
     // TODO: Could be uint64
     seek_s.offset = mp_obj_get_int(args[1]);
     seek_s.whence = SEEK_SET;
     if (n_args == 3) {
         seek_s.whence = mp_obj_get_int(args[2]);
     }

     // In POSIX, it's error to seek before end of stream, we enforce it here.
     if (seek_s.whence == SEEK_SET && seek_s.offset < 0) {
         mp_raise_OSError(MP_EINVAL);
     }

     int error;
     mp_uint_t res = stream_p->ioctl(args[0], MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &error);
     if (res == MP_STREAM_ERROR) {
         mp_raise_OSError(error);
     }

     // TODO: Could be uint64
     return mp_obj_new_int_from_uint(seek_s.offset);
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj, 2, 3, stream_seek);

 STATIC mp_obj_t stream_tell(mp_obj_t self) {
     mp_obj_t offset = MP_OBJ_NEW_SMALL_INT(0);
     mp_obj_t whence = MP_OBJ_NEW_SMALL_INT(SEEK_CUR);
     const mp_obj_t args[3] = {self, offset, whence};
     return stream_seek(3, args);
 }
 MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_tell_obj, stream_tell);

 STATIC mp_obj_t stream_flush(mp_obj_t self) {
     const mp_stream_p_t *stream_p = mp_get_stream_raise(self, MP_STREAM_OP_IOCTL);
     int error;
     mp_uint_t res = stream_p->ioctl(self, MP_STREAM_FLUSH, 0, &error);
     if (res == MP_STREAM_ERROR) {
         mp_raise_OSError(error);
     }
     return mp_const_none;
 }
 MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_flush_obj, stream_flush);

 STATIC mp_obj_t stream_ioctl(size_t n_args, const mp_obj_t *args) {
     const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL);

     mp_buffer_info_t bufinfo;
     uintptr_t val = 0;
     if (n_args > 2) {
         if (mp_get_buffer(args[2], &bufinfo, MP_BUFFER_WRITE)) {
             val = (uintptr_t)bufinfo.buf;
         } else {
             val = mp_obj_get_int_truncated(args[2]);
         }
     }

     int error;
     mp_uint_t res = stream_p->ioctl(args[0], mp_obj_get_int(args[1]), val, &error);
     if (res == MP_STREAM_ERROR) {
         mp_raise_OSError(error);
     }

     return mp_obj_new_int(res);
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_ioctl_obj, 2, 3, stream_ioctl);

 #if MICROPY_STREAMS_POSIX_API
 /*
  * POSIX-like functions
  *
  * These functions have POSIX-compatible signature (except for "void *stream"
  * first argument instead of "int fd"). They are useful to port existing
  * POSIX-compatible software to work with MicroPython streams.
  */

 // errno-like variable. If any of the functions below returned with error
 // status, this variable will contain error no.
 int mp_stream_errno;

 ssize_t mp_stream_posix_write(mp_obj_t stream, const void *buf, size_t len) {
     mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
     const mp_stream_p_t *stream_p = o->type->protocol;
     mp_uint_t out_sz = stream_p->write(stream, buf, len, &mp_stream_errno);
     if (out_sz == MP_STREAM_ERROR) {
         return -1;
     } else {
         return out_sz;
     }
 }

 ssize_t mp_stream_posix_read(mp_obj_t stream, void *buf, size_t len) {
     mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
     const mp_stream_p_t *stream_p = o->type->protocol;
     mp_uint_t out_sz = stream_p->read(stream, buf, len, &mp_stream_errno);
     if (out_sz == MP_STREAM_ERROR) {
         return -1;
     } else {
         return out_sz;
     }
 }

 off_t mp_stream_posix_lseek(mp_obj_t stream, off_t offset, int whence) {
     const mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
     const mp_stream_p_t *stream_p = o->type->protocol;
     struct mp_stream_seek_t seek_s;
     seek_s.offset = offset;
     seek_s.whence = whence;
     mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &mp_stream_errno);
     if (res == MP_STREAM_ERROR) {
         return -1;
     }
     return seek_s.offset;
 }

 int mp_stream_posix_fsync(mp_obj_t stream) {
     mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
     const mp_stream_p_t *stream_p = o->type->protocol;
     mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_FLUSH, 0, &mp_stream_errno);
     if (res == MP_STREAM_ERROR) {
         return -1;
     }
     return res;
 }

 #endif
	/*
	* This file is part of the MicroPython project, http://micropython.org/
	*
	* The MIT License (MIT)
	*
	* Copyright (c) 2013, 2014 Damien P. George
	* Copyright (c) 2014 Paul Sokolovsky
	*
	* Permission is hereby granted, free of charge, to any person obtaining a copy
	* of this software and associated documentation files (the "Software"), to deal
	* in the Software without restriction, including without limitation the rights
	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	* copies of the Software, and to permit persons to whom the Software is
	* furnished to do so, subject to the following conditions:
	*
	* The above copyright notice and this permission notice shall be included in
	* all copies or substantial portions of the Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
	* THE SOFTWARE.
	*/

	#include <string.h>
	#include <unistd.h>

	#include "py/objstr.h"
	#include "py/stream.h"
	#include "py/runtime.h"

	#if MICROPY_STREAMS_NON_BLOCK
	#include <errno.h>
	#if defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR)
	#define EWOULDBLOCK 140
	#endif
	#endif

	// This file defines generic Python stream read/write methods which
	// dispatch to the underlying stream interface of an object.

	// TODO: should be in mpconfig.h
	#define DEFAULT_BUFFER_SIZE 256

	STATIC mp_obj_t stream_readall(mp_obj_t self_in);

	#define STREAM_CONTENT_TYPE(stream) (((stream)->is_text) ? &mp_type_str : &mp_type_bytes)

	// Returns error condition in *errcode, if non-zero, return value is number of bytes written
	// before error condition occurred. If *errcode == 0, returns total bytes written (which will
	// be equal to input size).
	mp_uint_t mp_stream_rw(mp_obj_t stream, void buf_, mp_uint_t size, int errcode, byte flags) {
	byte *buf = buf_;
	mp_obj_base_t* s = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
	typedef mp_uint_t (io_func_t)(mp_obj_t obj, void buf, mp_uint_t size, int *errcode);
	io_func_t io_func;
	const mp_stream_p_t *stream_p = s->type->protocol;
	if (flags & MP_STREAM_RW_WRITE) {
	io_func = (io_func_t)stream_p->write;
	} else {
	io_func = stream_p->read;
	}

	*errcode = 0;
	mp_uint_t done = 0;
	while (size > 0) {
	mp_uint_t out_sz = io_func(stream, buf, size, errcode);
	// For read, out_sz == 0 means EOF. For write, it's unspecified
	// what it means, but we don't make any progress, so returning
	// is still the best option.
	if (out_sz == 0) {
	return done;
	}
	if (out_sz == MP_STREAM_ERROR) {
	// If we read something before getting EAGAIN, don't leak it
	if (mp_is_nonblocking_error(*errcode) && done != 0) {
	*errcode = 0;
	}
	return done;
	}
	if (flags & MP_STREAM_RW_ONCE) {
	return out_sz;
	}

	buf += out_sz;
	size -= out_sz;
	done += out_sz;
	}
	return done;
	}

	const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags) {
	mp_obj_type_t *type = mp_obj_get_type(self_in);
	const mp_stream_p_t *stream_p = type->protocol;
	if (stream_p == NULL
	\|\| ((flags & MP_STREAM_OP_READ) && stream_p->read == NULL)
	\|\| ((flags & MP_STREAM_OP_WRITE) && stream_p->write == NULL)
	\|\| ((flags & MP_STREAM_OP_IOCTL) && stream_p->ioctl == NULL)) {
	// CPython: io.UnsupportedOperation, OSError subclass
	mp_raise_msg(&mp_type_OSError, "stream operation not supported");
	}
	return stream_p;
	}

	mp_obj_t mp_stream_close(mp_obj_t stream) {
	// TODO: Still consider using ioctl for close
	mp_obj_t dest[2];
	mp_load_method(stream, MP_QSTR_close, dest);
	return mp_call_method_n_kw(0, 0, dest);
	}

	STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte flags) {
	const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ);

	// What to do if sz < -1? Python docs don't specify this case.
	// CPython does a readall, but here we silently let negatives through,
	// and they will cause a MemoryError.
	mp_int_t sz;
	if (n_args == 1 \|\| ((sz = mp_obj_get_int(args[1])) == -1)) {
	return stream_readall(args[0]);
	}

	#if MICROPY_PY_BUILTINS_STR_UNICODE
	if (stream_p->is_text) {
	// We need to read sz number of unicode characters. Because we don't have any
	// buffering, and because the stream API can only read bytes, we must read here
	// in units of bytes and must never over read. If we want sz chars, then reading
	// sz bytes will never over-read, so we follow this approach, in a loop to keep
	// reading until we have exactly enough chars. This will be 1 read for text
	// with ASCII-only chars, and about 2 reads for text with a couple of non-ASCII
	// chars. For text with lots of non-ASCII chars, it'll be pretty inefficient
	// in time and memory.

	vstr_t vstr;
	vstr_init(&vstr, sz);
	mp_uint_t more_bytes = sz;
	mp_uint_t last_buf_offset = 0;
	while (more_bytes > 0) {
	char *p = vstr_add_len(&vstr, more_bytes);
	int error;
	mp_uint_t out_sz = mp_stream_read_exactly(args[0], p, more_bytes, &error);
	if (error != 0) {
	vstr_cut_tail_bytes(&vstr, more_bytes);
	if (mp_is_nonblocking_error(error)) {
	// With non-blocking streams, we read as much as we can.
	// If we read nothing, return None, just like read().
	// Otherwise, return data read so far.
	// TODO what if we have read only half a non-ASCII char?
	if (vstr.len == 0) {
	vstr_clear(&vstr);
	return mp_const_none;
	}
	break;
	}
	mp_raise_OSError(error);
	}

	if (out_sz < more_bytes) {
	// Finish reading.
	// TODO what if we have read only half a non-ASCII char?
	vstr_cut_tail_bytes(&vstr, more_bytes - out_sz);
	if (out_sz == 0) {
	break;
	}
	}

	// count chars from bytes just read
	for (mp_uint_t off = last_buf_offset;;) {
	byte b = vstr.buf[off];
	int n;
	if (!UTF8_IS_NONASCII(b)) {
	// 1-byte ASCII char
	n = 1;
	} else if ((b & 0xe0) == 0xc0) {
	// 2-byte char
	n = 2;
	} else if ((b & 0xf0) == 0xe0) {
	// 3-byte char
	n = 3;
	} else if ((b & 0xf8) == 0xf0) {
	// 4-byte char
	n = 4;
	} else {
	// TODO
	n = 5;
	}
	if (off + n <= vstr.len) {
	// got a whole char in n bytes
	off += n;
	sz -= 1;
	last_buf_offset = off;
	if (off >= vstr.len) {
	more_bytes = sz;
	break;
	}
	} else {
	// didn't get a whole char, so work out how many extra bytes are needed for
	// this partial char, plus bytes for additional chars that we want
	more_bytes = (off + n - vstr.len) + (sz - 1);
	break;
	}
	}
	}

	return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);
	}
	#endif

	vstr_t vstr;
	vstr_init_len(&vstr, sz);
	int error;
	mp_uint_t out_sz = mp_stream_rw(args[0], vstr.buf, sz, &error, flags);
	if (error != 0) {
	vstr_clear(&vstr);
	if (mp_is_nonblocking_error(error)) {
	// https://docs.python.org/3.4/library/io.html#io.RawIOBase.read
	// "If the object is in non-blocking mode and no bytes are available,
	// None is returned."
	// This is actually very weird, as naive truth check will treat
	// this as EOF.
	return mp_const_none;
	}
	mp_raise_OSError(error);
	} else {
	vstr.len = out_sz;
	return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
	}
	}

	STATIC mp_obj_t stream_read(size_t n_args, const mp_obj_t *args) {
	return stream_read_generic(n_args, args, MP_STREAM_RW_READ);
	}
	MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read_obj, 1, 2, stream_read);

	STATIC mp_obj_t stream_read1(size_t n_args, const mp_obj_t *args) {
	return stream_read_generic(n_args, args, MP_STREAM_RW_READ \| MP_STREAM_RW_ONCE);
	}
	MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read1_obj, 1, 2, stream_read1);

	mp_obj_t mp_stream_write(mp_obj_t self_in, const void *buf, size_t len, byte flags) {
	mp_get_stream_raise(self_in, MP_STREAM_OP_WRITE);

	int error;
	mp_uint_t out_sz = mp_stream_rw(self_in, (void*)buf, len, &error, flags);
	if (error != 0) {
	if (mp_is_nonblocking_error(error)) {
	// http://docs.python.org/3/library/io.html#io.RawIOBase.write
	// "None is returned if the raw stream is set not to block and
	// no single byte could be readily written to it."
	return mp_const_none;
	}
	mp_raise_OSError(error);
	} else {
	return MP_OBJ_NEW_SMALL_INT(out_sz);
	}
	}

	// XXX hack
	void mp_stream_write_adaptor(void self, const char buf, size_t len) {
	mp_stream_write(MP_OBJ_FROM_PTR(self), buf, len, MP_STREAM_RW_WRITE);
	}

	STATIC mp_obj_t stream_write_method(size_t n_args, const mp_obj_t *args) {
	mp_buffer_info_t bufinfo;
	mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ);
	size_t max_len = (size_t)-1;
	size_t off = 0;
	if (n_args == 3) {
	max_len = mp_obj_get_int_truncated(args[2]);
	} else if (n_args == 4) {
	off = mp_obj_get_int_truncated(args[2]);
	max_len = mp_obj_get_int_truncated(args[3]);
	if (off > bufinfo.len) {
	off = bufinfo.len;
	}
	}
	bufinfo.len -= off;
	return mp_stream_write(args[0], (byte*)bufinfo.buf + off, MIN(bufinfo.len, max_len), MP_STREAM_RW_WRITE);
	}
	MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj, 2, 4, stream_write_method);

	STATIC mp_obj_t stream_write1_method(mp_obj_t self_in, mp_obj_t arg) {
	mp_buffer_info_t bufinfo;
	mp_get_buffer_raise(arg, &bufinfo, MP_BUFFER_READ);
	return mp_stream_write(self_in, bufinfo.buf, bufinfo.len, MP_STREAM_RW_WRITE \| MP_STREAM_RW_ONCE);
	}
	MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_write1_obj, stream_write1_method);

	STATIC mp_obj_t stream_readinto(size_t n_args, const mp_obj_t *args) {
	mp_get_stream_raise(args[0], MP_STREAM_OP_READ);
	mp_buffer_info_t bufinfo;
	mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_WRITE);

	// CPython extension: if 2nd arg is provided, that's max len to read,
	// instead of full buffer. Similar to
	// https://docs.python.org/3/library/socket.html#socket.socket.recv_into
	mp_uint_t len = bufinfo.len;
	if (n_args > 2) {
	len = mp_obj_get_int(args[2]);
	if (len > bufinfo.len) {
	len = bufinfo.len;
	}
	}

	int error;
	mp_uint_t out_sz = mp_stream_read_exactly(args[0], bufinfo.buf, len, &error);
	if (error != 0) {
	if (mp_is_nonblocking_error(error)) {
	return mp_const_none;
	}
	mp_raise_OSError(error);
	} else {
	return MP_OBJ_NEW_SMALL_INT(out_sz);
	}
	}
	MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto_obj, 2, 3, stream_readinto);

	STATIC mp_obj_t stream_readall(mp_obj_t self_in) {
	const mp_stream_p_t *stream_p = mp_get_stream_raise(self_in, MP_STREAM_OP_READ);

	mp_uint_t total_size = 0;
	vstr_t vstr;
	vstr_init(&vstr, DEFAULT_BUFFER_SIZE);
	char *p = vstr.buf;
	mp_uint_t current_read = DEFAULT_BUFFER_SIZE;
	while (true) {
	int error;
	mp_uint_t out_sz = stream_p->read(self_in, p, current_read, &error);
	if (out_sz == MP_STREAM_ERROR) {
	if (mp_is_nonblocking_error(error)) {
	// With non-blocking streams, we read as much as we can.
	// If we read nothing, return None, just like read().
	// Otherwise, return data read so far.
	if (total_size == 0) {
	return mp_const_none;
	}
	break;
	}
	mp_raise_OSError(error);
	}
	if (out_sz == 0) {
	break;
	}
	total_size += out_sz;
	if (out_sz < current_read) {
	current_read -= out_sz;
	p += out_sz;
	} else {
	p = vstr_extend(&vstr, DEFAULT_BUFFER_SIZE);
	current_read = DEFAULT_BUFFER_SIZE;
	}
	}

	vstr.len = total_size;
	return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
	}

	// Unbuffered, inefficient implementation of readline() for raw I/O files.
	STATIC mp_obj_t stream_unbuffered_readline(size_t n_args, const mp_obj_t *args) {
	const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ);

	mp_int_t max_size = -1;
	if (n_args > 1) {
	max_size = MP_OBJ_SMALL_INT_VALUE(args[1]);
	}

	vstr_t vstr;
	if (max_size != -1) {
	vstr_init(&vstr, max_size);
	} else {
	vstr_init(&vstr, 16);
	}

	while (max_size == -1 \|\| max_size-- != 0) {
	char *p = vstr_add_len(&vstr, 1);
	int error;
	mp_uint_t out_sz = stream_p->read(args[0], p, 1, &error);
	if (out_sz == MP_STREAM_ERROR) {
	if (mp_is_nonblocking_error(error)) {
	if (vstr.len == 1) {
	// We just incremented it, but otherwise we read nothing
	// and immediately got EAGAIN. This case is not well
	// specified in
	// https://docs.python.org/3/library/io.html#io.IOBase.readline
	// unlike similar case for read(). But we follow the latter's
	// behavior - return None.
	vstr_clear(&vstr);
	return mp_const_none;
	} else {
	goto done;
	}
	}
	mp_raise_OSError(error);
	}
	if (out_sz == 0) {
	done:
	// Back out previously added byte
	// Consider, what's better - read a char and get OutOfMemory (so read
	// char is lost), or allocate first as we do.
	vstr_cut_tail_bytes(&vstr, 1);
	break;
	}
	if (*p == '\n') {
	break;
	}
	}

	return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
	}
	MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj, 1, 2, stream_unbuffered_readline);

	// TODO take an optional extra argument (what does it do exactly?)
	STATIC mp_obj_t stream_unbuffered_readlines(mp_obj_t self) {
	mp_obj_t lines = mp_obj_new_list(0, NULL);
	for (;;) {
	mp_obj_t line = stream_unbuffered_readline(1, &self);
	if (!mp_obj_is_true(line)) {
	break;
	}
	mp_obj_list_append(lines, line);
	}
	return lines;
	}
	MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj, stream_unbuffered_readlines);

	mp_obj_t mp_stream_unbuffered_iter(mp_obj_t self) {
	mp_obj_t l_in = stream_unbuffered_readline(1, &self);
	if (mp_obj_is_true(l_in)) {
	return l_in;
	}
	return MP_OBJ_STOP_ITERATION;
	}

	STATIC mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) {
	const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL);

	struct mp_stream_seek_t seek_s;
	// TODO: Could be uint64
	seek_s.offset = mp_obj_get_int(args[1]);
	seek_s.whence = SEEK_SET;
	if (n_args == 3) {
	seek_s.whence = mp_obj_get_int(args[2]);
	}

	// In POSIX, it's error to seek before end of stream, we enforce it here.
	if (seek_s.whence == SEEK_SET && seek_s.offset < 0) {
	mp_raise_OSError(MP_EINVAL);
	}

	int error;
	mp_uint_t res = stream_p->ioctl(args[0], MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &error);
	if (res == MP_STREAM_ERROR) {
	mp_raise_OSError(error);
	}

	// TODO: Could be uint64
	return mp_obj_new_int_from_uint(seek_s.offset);
	}
	MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj, 2, 3, stream_seek);

	STATIC mp_obj_t stream_tell(mp_obj_t self) {
	mp_obj_t offset = MP_OBJ_NEW_SMALL_INT(0);
	mp_obj_t whence = MP_OBJ_NEW_SMALL_INT(SEEK_CUR);
	const mp_obj_t args[3] = {self, offset, whence};
	return stream_seek(3, args);
	}
	MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_tell_obj, stream_tell);

	STATIC mp_obj_t stream_flush(mp_obj_t self) {
	const mp_stream_p_t *stream_p = mp_get_stream_raise(self, MP_STREAM_OP_IOCTL);
	int error;
	mp_uint_t res = stream_p->ioctl(self, MP_STREAM_FLUSH, 0, &error);
	if (res == MP_STREAM_ERROR) {
	mp_raise_OSError(error);
	}
	return mp_const_none;
	}
	MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_flush_obj, stream_flush);

	STATIC mp_obj_t stream_ioctl(size_t n_args, const mp_obj_t *args) {
	const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL);

	mp_buffer_info_t bufinfo;
	uintptr_t val = 0;
	if (n_args > 2) {
	if (mp_get_buffer(args[2], &bufinfo, MP_BUFFER_WRITE)) {
	val = (uintptr_t)bufinfo.buf;
	} else {
	val = mp_obj_get_int_truncated(args[2]);
	}
	}

	int error;
	mp_uint_t res = stream_p->ioctl(args[0], mp_obj_get_int(args[1]), val, &error);
	if (res == MP_STREAM_ERROR) {
	mp_raise_OSError(error);
	}

	return mp_obj_new_int(res);
	}
	MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_ioctl_obj, 2, 3, stream_ioctl);

	#if MICROPY_STREAMS_POSIX_API
	/*
	* POSIX-like functions
	*
	* These functions have POSIX-compatible signature (except for "void *stream"
	* first argument instead of "int fd"). They are useful to port existing
	* POSIX-compatible software to work with MicroPython streams.
	*/

	// errno-like variable. If any of the functions below returned with error
	// status, this variable will contain error no.
	int mp_stream_errno;

	ssize_t mp_stream_posix_write(mp_obj_t stream, const void *buf, size_t len) {
	mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
	const mp_stream_p_t *stream_p = o->type->protocol;
	mp_uint_t out_sz = stream_p->write(stream, buf, len, &mp_stream_errno);
	if (out_sz == MP_STREAM_ERROR) {
	return -1;
	} else {
	return out_sz;
	}
	}

	ssize_t mp_stream_posix_read(mp_obj_t stream, void *buf, size_t len) {
	mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
	const mp_stream_p_t *stream_p = o->type->protocol;
	mp_uint_t out_sz = stream_p->read(stream, buf, len, &mp_stream_errno);
	if (out_sz == MP_STREAM_ERROR) {
	return -1;
	} else {
	return out_sz;
	}
	}

	off_t mp_stream_posix_lseek(mp_obj_t stream, off_t offset, int whence) {
	const mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
	const mp_stream_p_t *stream_p = o->type->protocol;
	struct mp_stream_seek_t seek_s;
	seek_s.offset = offset;
	seek_s.whence = whence;
	mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &mp_stream_errno);
	if (res == MP_STREAM_ERROR) {
	return -1;
	}
	return seek_s.offset;
	}

	int mp_stream_posix_fsync(mp_obj_t stream) {
	mp_obj_base_t* o = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream);
	const mp_stream_p_t *stream_p = o->type->protocol;
	mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_FLUSH, 0, &mp_stream_errno);
	if (res == MP_STREAM_ERROR) {
	return -1;
	}
	return res;
	}

	#endif