From dac3fc4840e8da03596aa5984c349c8df961ab7f Mon Sep 17 00:00:00 2001 From: Matti Pastell Date: Mon, 9 Jan 2017 17:49:28 +0200 Subject: [PATCH 01/15] Add chomp option to readline(s) --- base/io.jl | 82 +++++++++++++++++++++++++++++++++--------------- base/iostream.jl | 14 +++++++-- test/iobuffer.jl | 6 ++++ test/read.jl | 4 +++ 4 files changed, 78 insertions(+), 28 deletions(-) diff --git a/base/io.jl b/base/io.jl index ec855c2a5e17b..c6c7e54a48519 100644 --- a/base/io.jl +++ b/base/io.jl @@ -167,24 +167,50 @@ The text is assumed to be encoded in UTF-8. readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), filename) """ - readline(stream::IO=STDIN) - readline(filename::AbstractString) + readline(stream::IO=STDIN, chomp::Bool=false) + readline(filename::AbstractString, chomp::Bool=false) + +Read a single line of text from the given I/O stream or file (defaults to `STDIN`). +Lines in the input can end in `'\\n'` or `'\\r\\n'`. When reading from a file, the text is +assumed to be encoded in UTF-8. If `chomp=false` trailing newline character(s) will be included +in the output (if reached before the end of the input); otherwise newline characters(s) +are stripped from result. +""" +function readline(filename::AbstractString, chomp = false) + open(filename) do f + readline(f, chomp) + end +end -Read a single line of text, including a trailing newline character (if one is reached before -the end of the input), from the given I/O stream or file (defaults to `STDIN`). -When reading from a file, the text is assumed to be encoded in UTF-8. -""" -readline(filename::AbstractString) = open(readline, filename) +function readline(s::IO=STDIN, chomp::Bool=false) + chomp || return String(readuntil(s, 0x0a)) -""" - readlines(stream::IO) - readlines(filename::AbstractString) + line = readuntil(s, 0x0a) + i = length(line) + if i < 1 || line[i] != 0x0a + return String(line[1:i]) + elseif i < 2 || line[i-1] != 0x0d + return String(line[1:i-1]) + else + return String(line[1:i-2]) + end +end -Read all lines of an I/O stream or a file as a vector of strings. -The text is assumed to be encoded in UTF-8. """ -readlines(filename::AbstractString) = open(readlines, filename) + readlines(stream::IO, chomp::Bool=false) + readlines(filename::AbstractString, chomp::Bool=false) +Read all lines of an I/O stream or a file as a vector of strings. +Lines in the input can end in `'\\n'` or `'\\r\\n'`. +The text is assumed to be encoded in UTF-8. If `chomp=false` +trailing newline character(s) will be included in the output; +otherwise newline characters(s) are stripped from result. +""" +function readlines(filename::AbstractString, chomp::Bool=false) + open(filename) do f + readlines(f, chomp) + end +end ## byte-order mark, ntoh & hton ## @@ -447,9 +473,6 @@ function readuntil(s::IO, t::AbstractString) return String(take!(out)) end -readline() = readline(STDIN) -readline(s::IO) = readuntil(s, '\n') - """ readchomp(x) @@ -512,22 +535,28 @@ readstring(filename::AbstractString) = open(readstring, filename) type EachLine stream::IO + chomp::Bool ondone::Function - EachLine(stream) = EachLine(stream, ()->nothing) - EachLine(stream, ondone) = new(stream, ondone) + EachLine(stream, chomp) = EachLine(stream, chomp, ()->nothing) + EachLine(stream, chomp, ondone) = new(stream, chomp, ondone) end """ - eachline(stream::IO) - eachline(filename::AbstractString) + eachline(stream::IO, chomp::Bool=false) + eachline(filename::AbstractString, chomp::Bool=false) Create an iterable object that will yield each line from an I/O stream or a file. -The text is assumed to be encoded in UTF-8. +Lines in the input can end in `'\\n'` or `'\\r\\n'`. +The text is assumed to be encoded in UTF-8. If `chomp=false` +trailing newline character(s) will be included in the output; +otherwise newline characters(s) are stripped from result. """ -eachline(stream::IO) = EachLine(stream) -function eachline(filename::AbstractString) +eachline(stream::IO, chomp::Bool=false) = EachLine(stream, chomp) + + +function eachline(filename::AbstractString, chomp::Bool=false) s = open(filename) - EachLine(s, ()->close(s)) + EachLine(s, chomp, ()->close(s)) end start(itr::EachLine) = nothing @@ -538,10 +567,11 @@ function done(itr::EachLine, nada) itr.ondone() true end -next(itr::EachLine, nada) = (readline(itr.stream), nothing) + +next(itr::EachLine, nada) = (readline(itr.stream, itr.chomp), nothing) eltype(::Type{EachLine}) = String -readlines(s=STDIN) = collect(eachline(s)) +readlines(s::IO=STDIN, chomp::Bool=false) = collect(eachline(s, chomp)) iteratorsize(::Type{EachLine}) = SizeUnknown() diff --git a/base/iostream.jl b/base/iostream.jl index 7cbb3787bd9ce..ca414ca22b3aa 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -225,8 +225,18 @@ function readuntil(s::IOStream, delim::UInt8) ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8), s.ios, delim, 0) end -function readline(s::IOStream) - ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, '\n', 1) +function readline(s::IOStream, chomp::Bool=false) + chomp || return ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, '\n', 1) + + s = ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, '\n', 1) + i = endof(s) + if i < 1 || codeunit(s,i) != 0x0a + return s[1:i] + elseif i < 2 || codeunit(s,i-1) != 0x0d + return s[1:i-1] + else + return s[1:i-2] + end end function readbytes_all!(s::IOStream, b::Array{UInt8}, nb) diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 5d429a39317a2..4a24b96875a77 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -74,6 +74,12 @@ Base.compact(io) @test_throws ArgumentError seek(io,0) @test_throws ArgumentError truncate(io,0) @test readline(io) == "whipped cream\n" +@test write(io,"pancakes\nwaffles\nblueberries\n") > 0 +@test readlines(io, true) == String["pancakes", "waffles", "blueberries"] +write(io,"\n\r\n\n\r \n") > 0 +@test readlines(io, false) == String["\n", "\r\n", "\n", "\r \n"] +write(io,"\n\r\n\n\r \n") > 0 +@test readlines(io, true) == String["", "", "", "\r "] Base.compact(io) @test position(io) == 0 @test ioslength(io) == 0 diff --git a/test/read.jl b/test/read.jl index 42bce748e9538..d46104e7d05a1 100644 --- a/test/read.jl +++ b/test/read.jl @@ -247,8 +247,12 @@ for (name, f) in l verbose && println("$name readlines...") @test readlines(io()) == readlines(IOBuffer(text)) @test readlines(io()) == readlines(filename) + @test readlines(io(), true) == readlines(IOBuffer(text), true) + @test readlines(io(), true) == readlines(filename, true) @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) @test collect(eachline(io())) == collect(eachline(filename)) + @test collect(eachline(io(), true)) == collect(eachline(IOBuffer(text), true)) + @test collect(eachline(io(), true)) == collect(eachline(filename, true)) cleanup() From cdf10df4f145611c3073b76fc2e0812c87547de3 Mon Sep 17 00:00:00 2001 From: Matti Pastell Date: Tue, 10 Jan 2017 00:26:32 +0200 Subject: [PATCH 02/15] Implement chomp in C --- base/iostream.jl | 12 +----------- src/support/ios.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ src/support/ios.h | 1 + src/sys.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 11 deletions(-) diff --git a/base/iostream.jl b/base/iostream.jl index ca414ca22b3aa..4fe65bae8d63c 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -226,17 +226,7 @@ function readuntil(s::IOStream, delim::UInt8) end function readline(s::IOStream, chomp::Bool=false) - chomp || return ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, '\n', 1) - - s = ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, '\n', 1) - i = endof(s) - if i < 1 || codeunit(s,i) != 0x0a - return s[1:i] - elseif i < 2 || codeunit(s,i-1) != 0x0d - return s[1:i-1] - else - return s[1:i-2] - end + ccall(:jl_readline, Ref{String}, (Ptr{Void}, UInt8), s.ios, chomp) end function readbytes_all!(s::IOStream, b::Array{UInt8}, nb) diff --git a/src/support/ios.c b/src/support/ios.c index 650a8690c0888..12a628a7f5f5b 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -831,6 +831,50 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) return total; } +size_t ios_copyline(ios_t *to, ios_t *from, uint8_t chomp) +{ + + size_t total = 0, avail = (size_t)(from->size - from->bpos); + + while (!ios_eof(from)) { + if (avail == 0) { + avail = ios_readprep(from, LINE_CHUNK_SIZE); + if (avail == 0) + break; + } + size_t written; + char *n = (char*)memchr(from->buf+from->bpos, '\n', avail); + + if (n == NULL) { + written = ios_write(to, from->buf+from->bpos, avail); + from->bpos += avail; + total += written; + avail = 0; + } + else { + size_t nchomp = 0; + size_t ntowrite = n - (from->buf+from->bpos) + 1; + + if (chomp) { + if (ntowrite > 1 && from->buf[from->bpos+ntowrite - 2] == '\r') { + nchomp = 2; + } + else { + nchomp = 1; + } + } + + written = ios_write(to, from->buf+from->bpos, ntowrite - nchomp); + from->bpos += ntowrite; + total += written; + return total; + } + } + from->_eof = 1; + return total; +} + + static void _ios_init(ios_t *s) { // put all fields in a sane initial state diff --git a/src/support/ios.h b/src/support/ios.h index 6acd52fb01838..46ab2884fb2ef 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -95,6 +95,7 @@ JL_DLLEXPORT void ios_set_readonly(ios_t *s); JL_DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes); JL_DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from); JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim); +JL_DLLEXPORT size_t ios_copyline(ios_t *to, ios_t *from, uint8_t chomp); // ensure at least n bytes are buffered if possible. returns # available. JL_DLLEXPORT size_t ios_readprep(ios_t *from, size_t n); diff --git a/src/sys.c b/src/sys.c index c4d7efaafdf51..3e3edd905cdf6 100644 --- a/src/sys.c +++ b/src/sys.c @@ -292,6 +292,51 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str) return (jl_value_t*)a; } +JL_DLLEXPORT jl_value_t *jl_readline(ios_t *s, uint8_t chomp) +{ + jl_array_t *a; + // manually inlined common case + char *n = (char*)memchr(s->buf+s->bpos, '\n', (size_t)(s->size - s->bpos)); + if (n) { + size_t ntowrite = n-(s->buf+s->bpos)+1; + size_t nchomp = 0; + if (chomp) + { + if (ntowrite > 1 && s->buf[s->bpos+ntowrite - 2] == '\r') { + nchomp = 2; + } + else { + nchomp = 1; + } + } + jl_value_t *str = jl_pchar_to_string(s->buf + s->bpos, ntowrite - nchomp); + s->bpos += ntowrite; + return str; + } + else { + a = jl_alloc_array_1d(jl_array_uint8_type, 80); + ios_t dest; + ios_mem(&dest, 0); + ios_setbuf(&dest, (char*)a->data, 80, 0); + size_t n = ios_copyline(&dest, s, chomp); + if (dest.buf != a->data) { + a = jl_take_buffer(&dest); + } + else { +#ifdef STORE_ARRAY_LEN + a->length = n; +#endif + a->nrows = n; + ((char*)a->data)[n] = '\0'; + } + + JL_GC_PUSH1(&a); + jl_value_t *st = jl_array_to_string(a); + JL_GC_POP(); + return st; + } +} + JL_DLLEXPORT uint64_t jl_ios_get_nbyte_int(ios_t *s, const size_t n) { assert(n <= 8); From 46207211f659bb7de1f0ee988b31fcf984012458 Mon Sep 17 00:00:00 2001 From: Matti Pastell Date: Tue, 10 Jan 2017 00:38:31 +0200 Subject: [PATCH 03/15] Improve performance for readline(s::IO), fix docs --- base/io.jl | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/base/io.jl b/base/io.jl index c6c7e54a48519..c508398507d18 100644 --- a/base/io.jl +++ b/base/io.jl @@ -171,9 +171,11 @@ readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), readline(filename::AbstractString, chomp::Bool=false) Read a single line of text from the given I/O stream or file (defaults to `STDIN`). -Lines in the input can end in `'\\n'` or `'\\r\\n'`. When reading from a file, the text is -assumed to be encoded in UTF-8. If `chomp=false` trailing newline character(s) will be included -in the output (if reached before the end of the input); otherwise newline characters(s) +Lines in the input can end in `'\\n'` or `"\\r\\n"`. When reading from a file, the text is +assumed to be encoded in UTF-8. + +If `chomp=false` trailing newline character(s) will be included in the output +(if reached before the end of the input); otherwise newline characters(s) are stripped from result. """ function readline(filename::AbstractString, chomp = false) @@ -187,12 +189,12 @@ function readline(s::IO=STDIN, chomp::Bool=false) line = readuntil(s, 0x0a) i = length(line) - if i < 1 || line[i] != 0x0a - return String(line[1:i]) + if i == 0 || line[i] != 0x0a + return String(line) elseif i < 2 || line[i-1] != 0x0d - return String(line[1:i-1]) + return String(resize!(line,i-1)) else - return String(line[1:i-2]) + return String(resize!(line,i-2)) end end @@ -201,9 +203,10 @@ end readlines(filename::AbstractString, chomp::Bool=false) Read all lines of an I/O stream or a file as a vector of strings. -Lines in the input can end in `'\\n'` or `'\\r\\n'`. -The text is assumed to be encoded in UTF-8. If `chomp=false` -trailing newline character(s) will be included in the output; +Lines in the input can end in `'\\n'` or `"\\r\\n"`. +The text is assumed to be encoded in UTF-8. + +If `chomp=false` trailing newline character(s) will be included in the output; otherwise newline characters(s) are stripped from result. """ function readlines(filename::AbstractString, chomp::Bool=false) @@ -546,9 +549,10 @@ end eachline(filename::AbstractString, chomp::Bool=false) Create an iterable object that will yield each line from an I/O stream or a file. -Lines in the input can end in `'\\n'` or `'\\r\\n'`. -The text is assumed to be encoded in UTF-8. If `chomp=false` -trailing newline character(s) will be included in the output; +Lines in the input can end in `'\\n'` or `"\\r\\n"`. +The text is assumed to be encoded in UTF-8. + +If `chomp=false` trailing newline character(s) will be included in the output; otherwise newline characters(s) are stripped from result. """ eachline(stream::IO, chomp::Bool=false) = EachLine(stream, chomp) From 3590288556e35929d65767c831dd436060ef5166 Mon Sep 17 00:00:00 2001 From: Matti Pastell Date: Tue, 10 Jan 2017 09:48:26 +0200 Subject: [PATCH 04/15] Move chomp to jl_readuntil --- base/iostream.jl | 4 ++-- src/support/ios.c | 51 ++++++++++------------------------------------ src/support/ios.h | 4 ++-- src/sys.c | 52 ++++------------------------------------------- 4 files changed, 19 insertions(+), 92 deletions(-) diff --git a/base/iostream.jl b/base/iostream.jl index 4fe65bae8d63c..bd5ae9d94f310 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -222,11 +222,11 @@ take!(s::IOStream) = ccall(:jl_take_buffer, Vector{UInt8}, (Ptr{Void},), s.ios) function readuntil(s::IOStream, delim::UInt8) - ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8), s.ios, delim, 0) + ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 0, 0) end function readline(s::IOStream, chomp::Bool=false) - ccall(:jl_readline, Ref{String}, (Ptr{Void}, UInt8), s.ios, chomp) + ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, '\n', 1, chomp) end function readbytes_all!(s::IOStream, b::Array{UInt8}, nb) diff --git a/src/support/ios.c b/src/support/ios.c index 12a628a7f5f5b..6a3cc11825fc9 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -802,7 +802,7 @@ size_t ios_copyall(ios_t *to, ios_t *from) #define LINE_CHUNK_SIZE 160 -size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) +size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp) { size_t total = 0, avail = (size_t)(from->size - from->bpos); while (!ios_eof(from)) { @@ -821,7 +821,8 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) } else { size_t ntowrite = pd - (from->buf+from->bpos) + 1; - written = ios_write(to, from->buf+from->bpos, ntowrite); + size_t nchomp = ios_nchomp(from, ntowrite, chomp); + written = ios_write(to, from->buf+from->bpos, ntowrite - nchomp); from->bpos += ntowrite; total += written; return total; @@ -831,50 +832,20 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) return total; } -size_t ios_copyline(ios_t *to, ios_t *from, uint8_t chomp) +size_t ios_nchomp(ios_t *from, size_t ntowrite, uint8_t chomp) { - - size_t total = 0, avail = (size_t)(from->size - from->bpos); - - while (!ios_eof(from)) { - if (avail == 0) { - avail = ios_readprep(from, LINE_CHUNK_SIZE); - if (avail == 0) - break; - } - size_t written; - char *n = (char*)memchr(from->buf+from->bpos, '\n', avail); - - if (n == NULL) { - written = ios_write(to, from->buf+from->bpos, avail); - from->bpos += avail; - total += written; - avail = 0; + size_t nchomp = 0; + if (chomp) { + if (ntowrite > 1 && from->buf[from->bpos+ntowrite - 2] == '\r') { + nchomp = 2; } else { - size_t nchomp = 0; - size_t ntowrite = n - (from->buf+from->bpos) + 1; - - if (chomp) { - if (ntowrite > 1 && from->buf[from->bpos+ntowrite - 2] == '\r') { - nchomp = 2; - } - else { - nchomp = 1; - } - } - - written = ios_write(to, from->buf+from->bpos, ntowrite - nchomp); - from->bpos += ntowrite; - total += written; - return total; + nchomp = 1; } } - from->_eof = 1; - return total; + return nchomp; } - static void _ios_init(ios_t *s) { // put all fields in a sane initial state @@ -1179,7 +1150,7 @@ char *ios_readline(ios_t *s) { ios_t dest; ios_mem(&dest, 0); - ios_copyuntil(&dest, s, '\n'); + ios_copyuntil(&dest, s, '\n', 0); size_t n; return ios_take_buffer(&dest, &n); } diff --git a/src/support/ios.h b/src/support/ios.h index 46ab2884fb2ef..f901e74f21893 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -94,8 +94,8 @@ JL_DLLEXPORT int ios_get_writable(ios_t *s); JL_DLLEXPORT void ios_set_readonly(ios_t *s); JL_DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes); JL_DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from); -JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim); -JL_DLLEXPORT size_t ios_copyline(ios_t *to, ios_t *from, uint8_t chomp); +JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp); +JL_DLLEXPORT size_t ios_nchomp(ios_t *from, size_t ntowrite, uint8_t chomp); // ensure at least n bytes are buffered if possible. returns # available. JL_DLLEXPORT size_t ios_readprep(ios_t *from, size_t n); diff --git a/src/sys.c b/src/sys.c index 3e3edd905cdf6..49af7c42c925d 100644 --- a/src/sys.c +++ b/src/sys.c @@ -250,7 +250,7 @@ JL_DLLEXPORT jl_array_t *jl_take_buffer(ios_t *s) return a; } -JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str) +JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint8_t chomp) { jl_array_t *a; // manually inlined common case @@ -258,7 +258,8 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str) if (pd) { size_t n = pd-(s->buf+s->bpos)+1; if (str) { - jl_value_t *str = jl_pchar_to_string(s->buf + s->bpos, n); + size_t nchomp = ios_nchomp(s, n, chomp); + jl_value_t *str = jl_pchar_to_string(s->buf + s->bpos, n - nchomp); s->bpos += n; return str; } @@ -271,7 +272,7 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str) ios_t dest; ios_mem(&dest, 0); ios_setbuf(&dest, (char*)a->data, 80, 0); - size_t n = ios_copyuntil(&dest, s, delim); + size_t n = ios_copyuntil(&dest, s, delim, chomp); if (dest.buf != a->data) { a = jl_take_buffer(&dest); } @@ -292,51 +293,6 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str) return (jl_value_t*)a; } -JL_DLLEXPORT jl_value_t *jl_readline(ios_t *s, uint8_t chomp) -{ - jl_array_t *a; - // manually inlined common case - char *n = (char*)memchr(s->buf+s->bpos, '\n', (size_t)(s->size - s->bpos)); - if (n) { - size_t ntowrite = n-(s->buf+s->bpos)+1; - size_t nchomp = 0; - if (chomp) - { - if (ntowrite > 1 && s->buf[s->bpos+ntowrite - 2] == '\r') { - nchomp = 2; - } - else { - nchomp = 1; - } - } - jl_value_t *str = jl_pchar_to_string(s->buf + s->bpos, ntowrite - nchomp); - s->bpos += ntowrite; - return str; - } - else { - a = jl_alloc_array_1d(jl_array_uint8_type, 80); - ios_t dest; - ios_mem(&dest, 0); - ios_setbuf(&dest, (char*)a->data, 80, 0); - size_t n = ios_copyline(&dest, s, chomp); - if (dest.buf != a->data) { - a = jl_take_buffer(&dest); - } - else { -#ifdef STORE_ARRAY_LEN - a->length = n; -#endif - a->nrows = n; - ((char*)a->data)[n] = '\0'; - } - - JL_GC_PUSH1(&a); - jl_value_t *st = jl_array_to_string(a); - JL_GC_POP(); - return st; - } -} - JL_DLLEXPORT uint64_t jl_ios_get_nbyte_int(ios_t *s, const size_t n) { assert(n <= 8); From 2d07745f06ca6cb0af0215361226adcb7de20678 Mon Sep 17 00:00:00 2001 From: Matti Pastell Date: Tue, 10 Jan 2017 12:48:14 +0200 Subject: [PATCH 05/15] Deprecate readlines(s) and eachline, fix warnings --- base/LineEdit.jl | 4 +-- base/REPL.jl | 6 ++--- base/client.jl | 2 +- base/deprecated.jl | 12 +++++++++ base/io.jl | 25 ++++++++++--------- base/iostream.jl | 2 +- base/libgit2/callbacks.jl | 2 +- base/libgit2/utils.jl | 2 +- base/markdown/Common/block.jl | 22 ++++++++-------- base/markdown/GitHub/GitHub.jl | 4 +-- base/markdown/GitHub/table.jl | 2 +- base/markdown/parse/util.jl | 4 +-- base/multi.jl | 4 +-- base/pkg/dir.jl | 2 +- base/pkg/entry.jl | 2 +- base/precompile.jl | 4 ++- base/process.jl | 2 +- contrib/add_license_to_files.jl | 2 +- examples/clustermanager/0mq/ZMQCM.jl | 2 +- .../clustermanager/simple/UnixDomainCM.jl | 2 +- src/gen_sysimg_symtab.jl | 2 +- test/file.jl | 8 +++--- test/iobuffer.jl | 12 ++++----- test/iterators.jl | 4 +-- test/mmap.jl | 2 +- test/perf/kernel/actor_centrality.jl | 2 +- test/perf/kernel/perf.jl | 2 +- test/perf/perfcomp.jl | 2 +- test/perf/shootout/k_nucleotide.jl | 2 +- test/read.jl | 18 ++++++------- test/repl.jl | 20 +++++++-------- test/spawn.jl | 4 +-- 32 files changed, 100 insertions(+), 85 deletions(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index 4e6c4dfade98f..ae5bb5cb6bb66 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -211,7 +211,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf seek(buf, 0) moreinput = true # add a blank line if there is a trailing newline on the last line while moreinput - l = readline(buf) + l = readline(buf, false) moreinput = endswith(l, "\n") # We need to deal with on-screen characters, so use strwidth to compute occupied columns llength = strwidth(l) @@ -549,7 +549,7 @@ end function edit_kill_line(s::MIState) buf = buffer(s) pos = position(buf) - killbuf = readline(buf) + killbuf = readline(buf, false) if length(killbuf) > 1 && killbuf[end] == '\n' killbuf = killbuf[1:end-1] char_move_left(buf) diff --git a/base/REPL.jl b/base/REPL.jl index 8f21e231c31cb..9f78cd0168197 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -205,7 +205,7 @@ function run_frontend(repl::BasicREPL, backend::REPLBackendRef) interrupted = false while true try - line *= readline(repl.terminal) + line *= readline(repl.terminal, false) catch e if isa(e,InterruptException) try # raise the debugger if present @@ -337,7 +337,7 @@ An editor may have converted tabs to spaces at line """ function hist_getline(file) while !eof(file) - line = readline(file) + line = readline(file, false) isempty(line) && return line line[1] in "\r\n" || return line end @@ -995,7 +995,7 @@ function run_frontend(repl::StreamREPL, backend::REPLBackendRef) if have_color print(repl.stream, input_color(repl)) end - line = readline(repl.stream) + line = readline(repl.stream, false) if !isempty(line) ast = Base.parse_input_line(line) if have_color diff --git a/base/client.jl b/base/client.jl index 814a54ba7e9d6..b6c34e34d0a64 100644 --- a/base/client.jl +++ b/base/client.jl @@ -216,7 +216,7 @@ parse_input_line(s::AbstractString) = parse_input_line(String(s)) function parse_input_line(io::IO) s = "" while !eof(io) - s = s*readline(io) + s = s*readline(io, false) e = parse_input_line(s) if !(isa(e,Expr) && e.head === :incomplete) return e diff --git a/base/deprecated.jl b/base/deprecated.jl index 773ec6ae95bfb..190b61c050ab1 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1540,4 +1540,16 @@ unsafe_wrap(::Type{String}, p::Cstring, len::Integer, own::Bool=false) = # Rename LibGit2.GitAnyObject to LibGit2.GitUnknownObject (part of #19839) eval(LibGit2, :(Base.@deprecate_binding GitAnyObject GitUnknownObject)) + +# Deprecate one argument method in order to make chomp=true the new default #19944 +@deprecate readline(s::IO) readline(s, false) +@deprecate readline(s::IOStream) readline(s, false) +@deprecate readline(filename::AbstractString) readline(filename, false) +@deprecate readlines(s::IO) readlines(s, false) +@deprecate readlines(s::IOStream) readlines(s, false) +@deprecate readlines(filename::AbstractString) readlines(filename, false) +@deprecate eachline(s::IO) readlines(s, false) +@deprecate eachline(s::IOStream) readlines(s, false) +@deprecate eachline(filename::AbstractString) eachline(filename, false) + # End deprecations scheduled for 0.6 diff --git a/base/io.jl b/base/io.jl index c508398507d18..a39625b318b64 100644 --- a/base/io.jl +++ b/base/io.jl @@ -167,8 +167,8 @@ The text is assumed to be encoded in UTF-8. readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), filename) """ - readline(stream::IO=STDIN, chomp::Bool=false) - readline(filename::AbstractString, chomp::Bool=false) + readline(stream::IO=STDIN, chomp::Bool=true) + readline(filename::AbstractString, chomp::Bool=true) Read a single line of text from the given I/O stream or file (defaults to `STDIN`). Lines in the input can end in `'\\n'` or `"\\r\\n"`. When reading from a file, the text is @@ -178,13 +178,14 @@ If `chomp=false` trailing newline character(s) will be included in the output (if reached before the end of the input); otherwise newline characters(s) are stripped from result. """ -function readline(filename::AbstractString, chomp = false) +function readline(filename::AbstractString, chomp::Bool=true) open(filename) do f readline(f, chomp) end end +readline() = readline(STDIN, false) -function readline(s::IO=STDIN, chomp::Bool=false) +function readline(s::IO, chomp::Bool=true) chomp || return String(readuntil(s, 0x0a)) line = readuntil(s, 0x0a) @@ -199,8 +200,8 @@ function readline(s::IO=STDIN, chomp::Bool=false) end """ - readlines(stream::IO, chomp::Bool=false) - readlines(filename::AbstractString, chomp::Bool=false) + readlines(stream::IO, chomp::Bool=true) + readlines(filename::AbstractString, chomp::Bool=true) Read all lines of an I/O stream or a file as a vector of strings. Lines in the input can end in `'\\n'` or `"\\r\\n"`. @@ -209,7 +210,7 @@ The text is assumed to be encoded in UTF-8. If `chomp=false` trailing newline character(s) will be included in the output; otherwise newline characters(s) are stripped from result. """ -function readlines(filename::AbstractString, chomp::Bool=false) +function readlines(filename::AbstractString, chomp::Bool=true) open(filename) do f readlines(f, chomp) end @@ -545,8 +546,8 @@ type EachLine end """ - eachline(stream::IO, chomp::Bool=false) - eachline(filename::AbstractString, chomp::Bool=false) + eachline(stream::IO, chomp::Bool=true) + eachline(filename::AbstractString, chomp::Bool=true) Create an iterable object that will yield each line from an I/O stream or a file. Lines in the input can end in `'\\n'` or `"\\r\\n"`. @@ -555,10 +556,10 @@ The text is assumed to be encoded in UTF-8. If `chomp=false` trailing newline character(s) will be included in the output; otherwise newline characters(s) are stripped from result. """ -eachline(stream::IO, chomp::Bool=false) = EachLine(stream, chomp) +eachline(stream::IO, chomp::Bool=true) = EachLine(stream, chomp) -function eachline(filename::AbstractString, chomp::Bool=false) +function eachline(filename::AbstractString, chomp::Bool=true) s = open(filename) EachLine(s, chomp, ()->close(s)) end @@ -575,7 +576,7 @@ end next(itr::EachLine, nada) = (readline(itr.stream, itr.chomp), nothing) eltype(::Type{EachLine}) = String -readlines(s::IO=STDIN, chomp::Bool=false) = collect(eachline(s, chomp)) +readlines(s::IO=STDIN, chomp::Bool=true) = collect(eachline(s, chomp)) iteratorsize(::Type{EachLine}) = SizeUnknown() diff --git a/base/iostream.jl b/base/iostream.jl index bd5ae9d94f310..151ac63ab7f93 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -320,7 +320,7 @@ function skipchars(s::IOStream, pred; linecomment::Char=Char(0xffffffff)) ch = peekchar(s); status = Int(ch) while status >= 0 && (pred(ch) || ch == linecomment) if ch == linecomment - readline(s) + readline(s, false) else read(s, Char) # advance one character end diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 5c3543606e7c2..52da4edb28310 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -99,7 +99,7 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, else # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" open(privatekey) do f - passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED") + passphrase_required = (readline(f, false); readline(f, true) == "Proc-Type: 4,ENCRYPTED") end end diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl index f4bb35b55da6f..a0e4649b38348 100644 --- a/base/libgit2/utils.jl +++ b/base/libgit2/utils.jl @@ -31,7 +31,7 @@ function prompt(msg::AbstractString; default::AbstractString="", password::Bool= Base.getpass(msg) else print(msg) - chomp(readline(STDIN)) + readline(STDIN, true) end isempty(uinput) ? default : uinput end diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index edc773082722b..b79c5f28eb8aa 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -61,7 +61,7 @@ function hashheader(stream::IO, md::MD) return false if c != '\n' # Empty header - h = readline(stream) |> strip + h = readline(stream, false) |> strip h = match(r"(.*?)( +#+)?$", h).captures[1] buffer = IOBuffer() print(buffer, h) @@ -76,11 +76,11 @@ end function setextheader(stream::IO, md::MD) withstream(stream) do eatindent(stream) || return false - header = readline(stream) |> strip + header = readline(stream, false) |> strip header == "" && return false eatindent(stream) || return false - underline = readline(stream) |> strip + underline = readline(stream, false) |> strip length(underline) < 3 && return false u = underline[1] u in "-=" || return false @@ -108,7 +108,7 @@ function indentcode(stream::IO, block::MD) buffer = IOBuffer() while !eof(stream) if startswith(stream, " ") || startswith(stream, "\t") - write(buffer, readline(stream)) + write(buffer, readline(stream, false)) elseif blankline(stream) write(buffer, '\n') else @@ -139,10 +139,10 @@ function footnote(stream::IO, block::MD) else ref = match(regex, str).captures[1] buffer = IOBuffer() - write(buffer, readline(stream)) + write(buffer, readline(stream, false)) while !eof(stream) if startswith(stream, " ") - write(buffer, readline(stream)) + write(buffer, readline(stream, false)) elseif blankline(stream) write(buffer, '\n') else @@ -174,7 +174,7 @@ function blockquote(stream::IO, block::MD) empty = true while eatindent(stream) && startswith(stream, '>') startswith(stream, " ") - write(buffer, readline(stream)) + write(buffer, readline(stream, false)) empty = false end empty && return false @@ -210,7 +210,7 @@ function admonition(stream::IO, block::MD) category, title = let untitled = r"^([a-z]+)$", # !!! titled = r"^([a-z]+) \"(.*)\"$", # !!! "" - line = strip(readline(stream)) + line = strip(readline(stream, false)) if ismatch(untitled, line) m = match(untitled, line) # When no title is provided we use CATEGORY_NAME, capitalising it. @@ -229,7 +229,7 @@ function admonition(stream::IO, block::MD) buffer = IOBuffer() while !eof(stream) if startswith(stream, " ") - write(buffer, readline(stream)) + write(buffer, readline(stream, false)) elseif blankline(stream) write(buffer, '\n') else @@ -305,7 +305,7 @@ function list(stream::IO, block::MD) newline = false if startswith(stream, " "^indent) # Indented text that is part of the current list item. - print(buffer, readline(stream)) + print(buffer, readline(stream, false)) else matched = startswith(stream, regex) if isempty(matched) @@ -316,7 +316,7 @@ function list(stream::IO, block::MD) # Start of a new list item. count += 1 count > 1 && pushitem!(list, buffer) - print(buffer, readline(stream)) + print(buffer, readline(stream, false)) end end end diff --git a/base/markdown/GitHub/GitHub.jl b/base/markdown/GitHub/GitHub.jl index 591634d8e50b5..7e4b8c2263564 100644 --- a/base/markdown/GitHub/GitHub.jl +++ b/base/markdown/GitHub/GitHub.jl @@ -8,7 +8,7 @@ function fencedcode(stream::IO, block::MD) startswith(stream, "~~~", padding = true) || startswith(stream, "```", padding = true) || return false skip(stream, -1) ch = read(stream, Char) - trailing = strip(readline(stream)) + trailing = strip(readline(stream, false)) flavor = lstrip(trailing, ch) n = 3 + length(trailing) - length(flavor) @@ -30,7 +30,7 @@ function fencedcode(stream::IO, block::MD) seek(stream, line_start) end end - write(buffer, readline(stream)) + write(buffer, readline(stream, false)) end return false end diff --git a/base/markdown/GitHub/table.jl b/base/markdown/GitHub/table.jl index b310742c94068..32d30ed98cf42 100644 --- a/base/markdown/GitHub/table.jl +++ b/base/markdown/GitHub/table.jl @@ -7,7 +7,7 @@ end function parserow(stream::IO) withstream(stream) do - line = readline(stream) |> chomp + line = readline(stream, true) row = split(line, r"(?<!\\)\|") length(row) == 1 && return row[1] == "" && shift!(row) diff --git a/base/markdown/parse/util.jl b/base/markdown/parse/util.jl index 7ac42cda629f6..8bbec8ce9d0b7 100644 --- a/base/markdown/parse/util.jl +++ b/base/markdown/parse/util.jl @@ -46,7 +46,7 @@ function linecontains(io::IO, chars; allow_whitespace = true, eat = true, allowempty = false) start = position(io) - l = readline(io) |> chomp + l = readline(io, true) length(l) == 0 && return allowempty result = allowempty @@ -99,7 +99,7 @@ function startswith(stream::IO, r::Regex; eat = true, padding = false) @assert Base.startswith(r.pattern, "^") start = position(stream) padding && skipwhitespace(stream) - line = chomp(readline(stream)) + line = readline(stream, true) seek(stream, start) m = match(r, line) m === nothing && return "" diff --git a/base/multi.jl b/base/multi.jl index b60019ee1c5b7..4860ad44cf3b7 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -1624,7 +1624,7 @@ end function redirect_worker_output(ident, stream) @schedule while !eof(stream) - line = readline(stream) + line = readline(stream, false) if startswith(line, "\tFrom worker ") # STDOUT's of "additional" workers started from an initial worker on a host are not available # on the master directly - they are routed via the initial worker's STDOUT. @@ -1642,7 +1642,7 @@ end # setup a all-to-all network. function read_worker_host_port(io::IO) while true - conninfo = readline(io) + conninfo = readline(io, false) bind_addr, port = parse_connection_info(conninfo) if bind_addr != "" return bind_addr, port diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index ef38251c99f88..fe230e884fec3 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -67,7 +67,7 @@ end function getmetabranch() try - chomp(readline(joinpath(path(),"META_BRANCH"))) + readline(joinpath(path(),"META_BRANCH"), true) catch err META_BRANCH end diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 529cfdd44d83c..c16337cf32e79 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -608,7 +608,7 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) empty!(Base.DL_LOAD_PATH) append!(Base.DL_LOAD_PATH, $(repr(Base.DL_LOAD_PATH))) open("$(escape_string(errfile))", "a") do f - for path_ in eachline(STDIN) + for path_ in eachline(STDIN, false) path = chomp(path_) pkg = basename(dirname(dirname(path))) try diff --git a/base/precompile.jl b/base/precompile.jl index c8ce400142c17..82739d9ab6d04 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -322,7 +322,9 @@ precompile(Base.read, (Base.Terminals.TTYTerminal, Type{Char})) precompile(Base.read, (IOBuffer, Type{Char})) precompile(Base.read, (IOBuffer, Type{UInt8})) precompile(Base.read, (IOStream, Array{UInt32,1})) -precompile(Base.readline, (String,)) +precompile(Base.readline, (String, Bool)) +precompile(Base.readline, (IOBuffer, Bool)) +precompile(Base.readline, (IOStream, Bool)) precompile(Base.readuntil, (IOBuffer, Char)) precompile(Base.readuntil, (IOBuffer, UInt8)) precompile(Base.rehash!, (Dict{Any,Any}, Int)) diff --git a/base/process.jl b/base/process.jl index c8a249d858311..0cd1836e9980a 100644 --- a/base/process.jl +++ b/base/process.jl @@ -561,7 +561,7 @@ function eachline(cmd::AbstractCmd, stdin) close(stdout.in) out = stdout.out # implicitly close after reading lines, since we opened - return EachLine(out, ()->(close(out); success(processes) || pipeline_error(processes))) + return EachLine(out, false, ()->(close(out); success(processes) || pipeline_error(processes))) end eachline(cmd::AbstractCmd) = eachline(cmd, DevNull) diff --git a/contrib/add_license_to_files.jl b/contrib/add_license_to_files.jl index 63314844937af..ef52d3dfc4bc6 100644 --- a/contrib/add_license_to_files.jl +++ b/contrib/add_license_to_files.jl @@ -135,7 +135,7 @@ function add_license_line!(unprocessed::Vector, src::AbstractString, new_license if ext in keys(ext_prefix) prefix = ext_prefix[ext] f = open(path, "r") - lines = readlines(f) + lines = readlines(f, false) close(f) isempty(lines) && (push!(unprocessed, path); continue) isempty(old_license) || check_lines!(path, lines, old_license, prefix, true) diff --git a/examples/clustermanager/0mq/ZMQCM.jl b/examples/clustermanager/0mq/ZMQCM.jl index 1f305939b2ca5..da6421761cbe8 100644 --- a/examples/clustermanager/0mq/ZMQCM.jl +++ b/examples/clustermanager/0mq/ZMQCM.jl @@ -270,7 +270,7 @@ end function print_worker_stdout(io, pid) @schedule while !eof(io) - line = readline(io) + line = readline(io, false) print("\tFrom worker $(pid):\t$line") end end diff --git a/examples/clustermanager/simple/UnixDomainCM.jl b/examples/clustermanager/simple/UnixDomainCM.jl index b9519cde60caa..d2fd0e3248165 100644 --- a/examples/clustermanager/simple/UnixDomainCM.jl +++ b/examples/clustermanager/simple/UnixDomainCM.jl @@ -81,7 +81,7 @@ end function print_worker_stdout(io, pid) @schedule while !eof(io) - line = readline(io) + line = readline(io, false) print("\tFrom worker $(pid):\t$line") end end diff --git a/src/gen_sysimg_symtab.jl b/src/gen_sysimg_symtab.jl index bc8d79f516340..44f708752eab6 100644 --- a/src/gen_sysimg_symtab.jl +++ b/src/gen_sysimg_symtab.jl @@ -21,7 +21,7 @@ function outputline(io, line) println(io, "jl_symbol(\"", row[2], "\"),") end -lines = eachline(io) +lines = eachline(io, false) open(f->foreach(l->outputline(f,l), take(lines,63)), "common_symbols1.inc", "w") open(f->foreach(l->outputline(f,l), lines), "common_symbols2.inc", "w") diff --git a/test/file.jl b/test/file.jl index f8a7620e5ad51..8c27420774869 100644 --- a/test/file.jl +++ b/test/file.jl @@ -277,15 +277,15 @@ write(s, "Goodbye world!\n") close(s) s = open(file) mark(s) -str = readline(s) +str = readline(s, false) @test startswith(str, "Marked!") @test ismarked(s) reset(s) @test !ismarked(s) -str = readline(s) +str = readline(s, false) @test startswith(str, "Marked!") mark(s) -@test readline(s) == "Hello world!\n" +@test readline(s, false) == "Hello world!\n" @test ismarked(s) unmark(s) @test !ismarked(s) @@ -335,7 +335,7 @@ end emptyfile = joinpath(dir, "empty") touch(emptyfile) emptyf = open(emptyfile) -@test isempty(readlines(emptyf)) +@test isempty(readlines(emptyf, false)) close(emptyf) rm(emptyfile) diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 4a24b96875a77..459a76bd57112 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -49,7 +49,7 @@ end let io = IOBuffer("hamster\nguinea pig\nturtle") @test position(io) == 0 -@test readline(io) == "hamster\n" +@test readline(io, false) == "hamster\n" @test readstring(io) == "guinea pig\nturtle" @test_throws EOFError read(io,UInt8) seek(io,0) @@ -66,14 +66,14 @@ let io = PipeBuffer() @test_throws EOFError read(io,UInt8) @test write(io,"pancakes\nwaffles\nblueberries\n") > 0 @test position(io) == 0 -@test readline(io) == "pancakes\n" +@test readline(io, false) == "pancakes\n" Base.compact(io) -@test readline(io) == "waffles\n" +@test readline(io, false) == "waffles\n" @test write(io,"whipped cream\n") > 0 -@test readline(io) == "blueberries\n" +@test readline(io, false) == "blueberries\n" @test_throws ArgumentError seek(io,0) @test_throws ArgumentError truncate(io,0) -@test readline(io) == "whipped cream\n" +@test readline(io, false) == "whipped cream\n" @test write(io,"pancakes\nwaffles\nblueberries\n") > 0 @test readlines(io, true) == String["pancakes", "waffles", "blueberries"] write(io,"\n\r\n\n\r \n") > 0 @@ -139,7 +139,7 @@ a = Array{UInt8}(1024) @test eof(io) end -@test isempty(readlines(IOBuffer())) +@test isempty(readlines(IOBuffer(), false)) # issue #8193 let io=IOBuffer("asdf") diff --git a/test/iterators.jl b/test/iterators.jl index cfee13d484ec6..8ea0e0b3692c5 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -27,7 +27,7 @@ end # enumerate (issue #6284) let b = IOBuffer("1\n2\n3\n"), a = [] - for (i,x) in enumerate(eachline(b)) + for (i,x) in enumerate(eachline(b, false)) push!(a, (i,x)) end @test a == [(1,"1\n"),(2,"2\n"),(3,"3\n")] @@ -37,7 +37,7 @@ end let zeb = IOBuffer("1\n2\n3\n4\n5\n"), letters = ['a', 'b', 'c', 'd', 'e'], res = [] - for (number, letter) in zip(eachline(zeb), letters) + for (number, letter) in zip(eachline(zeb, false), letters) push!(res, (parse(Int,strip(number)), letter)) end @test res == [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')] diff --git a/test/mmap.jl b/test/mmap.jl index f79a1154baf21..b8eae23c72d60 100644 --- a/test/mmap.jl +++ b/test/mmap.jl @@ -130,7 +130,7 @@ c[5] = UInt8('x') Mmap.sync!(c) close(s) s = open(file, "r") -str = readline(s) +str = readline(s, false) close(s) @test startswith(str, "Hellx World") finalize(c); c=nothing; gc() diff --git a/test/perf/kernel/actor_centrality.jl b/test/perf/kernel/actor_centrality.jl index 4bc1f453742ab..d48a08b2a1171 100644 --- a/test/perf/kernel/actor_centrality.jl +++ b/test/perf/kernel/actor_centrality.jl @@ -43,7 +43,7 @@ function read_graph() open(joinpath(@__DIR__, "imdb-1.tsv"), "r") do io while !eof(io) - k = split(strip(readline(io)), "\t") + k = split(strip(readline(io), false), "\t") actor, movie = k[1], join(k[2:3], "_") ac, mn = get(G, actor), get(G, movie) push!(actors, actor) diff --git a/test/perf/kernel/perf.jl b/test/perf/kernel/perf.jl index b5e50514288cc..ac051be0d5f9c 100644 --- a/test/perf/kernel/perf.jl +++ b/test/perf/kernel/perf.jl @@ -105,7 +105,7 @@ d = randn(len) writecsv("random.csv", rand(100000,4)) function parsecsv() - for line in EachLine(open("random.csv")) + for line in eachline("random.csv", false) split(line, ',') end end diff --git a/test/perf/perfcomp.jl b/test/perf/perfcomp.jl index 7575107d90948..7723b2cb1b665 100644 --- a/test/perf/perfcomp.jl +++ b/test/perf/perfcomp.jl @@ -7,7 +7,7 @@ # The file format is the output of running `make` in this directory. function readperf(f) - Dict(rstrip(l[1:19])=>[parse(Float64,l[20:27]),parse(Float64,l[29:36]),parse(Float64,l[38:45]),parse(Float64,l[47:54])] for l in eachline(f)) + Dict(rstrip(l[1:19])=>[parse(Float64,l[20:27]),parse(Float64,l[29:36]),parse(Float64,l[38:45]),parse(Float64,l[47:54])] for l in eachline(f, false)) end function main() diff --git a/test/perf/shootout/k_nucleotide.jl b/test/perf/shootout/k_nucleotide.jl index 97f39624ff99d..c93d5708e0f67 100644 --- a/test/perf/shootout/k_nucleotide.jl +++ b/test/perf/shootout/k_nucleotide.jl @@ -64,7 +64,7 @@ function k_nucleotide(infile="knucleotide-input.txt") input = open(infile, "r") three = ">THREE " while true - line = readline(input) + line = readline(input, false) if length(line) >= length(three) && line[1:length(three)] == three break end diff --git a/test/read.jl b/test/read.jl index d46104e7d05a1..bcd3435682b4e 100644 --- a/test/read.jl +++ b/test/read.jl @@ -241,16 +241,16 @@ for (name, f) in l cleanup() verbose && println("$name readline...") - @test readline(io()) == readline(IOBuffer(text)) - @test readline(io()) == readline(filename) + @test readline(io(), false) == readline(IOBuffer(text), false) + @test readline(io(), false) == readline(filename, false) verbose && println("$name readlines...") - @test readlines(io()) == readlines(IOBuffer(text)) - @test readlines(io()) == readlines(filename) + @test readlines(io(), false) == readlines(IOBuffer(text), false) + @test readlines(io(), false) == readlines(filename, false) @test readlines(io(), true) == readlines(IOBuffer(text), true) @test readlines(io(), true) == readlines(filename, true) - @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) - @test collect(eachline(io())) == collect(eachline(filename)) + @test collect(eachline(io(), false)) == collect(eachline(IOBuffer(text), false)) + @test collect(eachline(io(), false)) == collect(eachline(filename, false)) @test collect(eachline(io(), true)) == collect(eachline(IOBuffer(text), true)) @test collect(eachline(io(), true)) == collect(eachline(filename, true)) @@ -275,13 +275,13 @@ for (name, f) in l verbose && println("$name seek...") for n = 0:length(text)-1 - @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) + @test readlines(seek(io(), n), false) == readlines(seek(IOBuffer(text), n), false) cleanup() end verbose && println("$name skip...") for n = 0:length(text)-1 - @test readlines(seek(io(), n)) == readlines(seek(IOBuffer(text), n)) - @test readlines(skip(io(), n)) == readlines(skip(IOBuffer(text), n)) + @test readlines(seek(io(), n), false) == readlines(seek(IOBuffer(text), n), false) + @test readlines(skip(io(), n), false) == readlines(skip(IOBuffer(text), n), false) cleanup() end verbose && println("$name seekend...") diff --git a/test/repl.jl b/test/repl.jl index 3bb48c368310f..11aeced63542d 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -103,7 +103,7 @@ if !is_windows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER # Test that accepting a REPL result immediately shows up, not # just on the next keystroke write(stdin_write, "1+1\n") # populate history with a trivial input - readline(stdout_read) + readline(stdout_read, false) write(stdin_write, "\e[A\n") t = Timer(10) do t isopen(t) || return @@ -116,19 +116,19 @@ if !is_windows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER # Issue #10222 # Test ignoring insert key in standard and prefix search modes write(stdin_write, "\e[2h\e[2h\n") # insert (VT100-style) - @test search(readline(stdout_read), "[2h") == 0:-1 - readline(stdout_read) + @test search(readline(stdout_read, false), "[2h") == 0:-1 + readline(stdout_read, false) write(stdin_write, "\e[2~\e[2~\n") # insert (VT220-style) - @test search(readline(stdout_read), "[2~") == 0:-1 - readline(stdout_read) + @test search(readline(stdout_read, false), "[2~") == 0:-1 + readline(stdout_read, false) write(stdin_write, "1+1\n") # populate history with a trivial input - readline(stdout_read) + readline(stdout_read, false) write(stdin_write, "\e[A\e[2h\n") # up arrow, insert (VT100-style) - readline(stdout_read) - readline(stdout_read) + readline(stdout_read, false) + readline(stdout_read, false) write(stdin_write, "\e[A\e[2~\n") # up arrow, insert (VT220-style) - readline(stdout_read) - readline(stdout_read) + readline(stdout_read, false) + readline(stdout_read, false) # Test down arrow to go back to history # populate history with a trivial input diff --git a/test/spawn.jl b/test/spawn.jl index d788ae7bf9547..b499038423507 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -182,8 +182,8 @@ let r, t, sock sock = connect(fetch(r)) mark(sock) @test ismarked(sock) - @test readline(sock) == "Hello, world!\n" - @test readline(sock) == "Goodbye, world...\n" + @test readline(sock, false) == "Hello, world!\n" + @test readline(sock, false) == "Goodbye, world...\n" @test reset(sock) == 0 @test !ismarked(sock) mark(sock) From 98fa88c4ef0645291b61bc19d35d10023c4e5e9f Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Tue, 10 Jan 2017 13:20:22 +0200 Subject: [PATCH 06/15] Fix deprecation warnings, clean whitespace --- base/deprecated.jl | 1 - base/io.jl | 2 +- test/spawn.jl | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 190b61c050ab1..c2ab600b80071 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1540,7 +1540,6 @@ unsafe_wrap(::Type{String}, p::Cstring, len::Integer, own::Bool=false) = # Rename LibGit2.GitAnyObject to LibGit2.GitUnknownObject (part of #19839) eval(LibGit2, :(Base.@deprecate_binding GitAnyObject GitUnknownObject)) - # Deprecate one argument method in order to make chomp=true the new default #19944 @deprecate readline(s::IO) readline(s, false) @deprecate readline(s::IOStream) readline(s, false) diff --git a/base/io.jl b/base/io.jl index a39625b318b64..19dc69bba57cf 100644 --- a/base/io.jl +++ b/base/io.jl @@ -212,7 +212,7 @@ otherwise newline characters(s) are stripped from result. """ function readlines(filename::AbstractString, chomp::Bool=true) open(filename) do f - readlines(f, chomp) + readlines(f, chomp) end end diff --git a/test/spawn.jl b/test/spawn.jl index b499038423507..e89fbcc82c310 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -188,12 +188,12 @@ let r, t, sock @test !ismarked(sock) mark(sock) @test ismarked(sock) - @test readline(sock) == "Hello, world!\n" + @test readline(sock, false) == "Hello, world!\n" unmark(sock) @test !ismarked(sock) @test_throws ArgumentError reset(sock) @test !unmark(sock) - @test readline(sock) == "Goodbye, world...\n" + @test readline(sock, true) == "Goodbye, world..." #@test eof(sock) ## doesn't work close(sock) @test wait(t) @@ -324,8 +324,8 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test c == read!(out, c) Base.wait_readnb(out, 1) @test nb_available(out) > 0 - ln1 = readline(out) - ln2 = readline(out) + ln1 = readline(out, false) + ln2 = readline(out, false) desc = readstring(out) @test !isreadable(out) @test !iswritable(out) From 88f32eba4cbfc70a0403b11654969c2da983df82 Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Tue, 10 Jan 2017 14:00:39 +0200 Subject: [PATCH 07/15] Fix build, update news --- NEWS.md | 3 +++ src/flisp/iostream.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 81db6bd8a8609..483e3cce24953 100644 --- a/NEWS.md +++ b/NEWS.md @@ -165,6 +165,9 @@ Compiler/Runtime improvements Deprecated or removed --------------------- + * One argument methods to `readline`, `readlines` and `eachline` have been depracated in + order to add new `chomp::Bool` argument which will default to `true` in future versions ([#19944]). + * `isdefined(a::Array, i::Int)` has been deprecated in favor of `isassigned` ([#18346]). * `is` has been deprecated in favor of `===` (which used to be an alias for `is`) ([#17758]). diff --git a/src/flisp/iostream.c b/src/flisp/iostream.c index efcbd024f3ccf..e3cb9170770fa 100644 --- a/src/flisp/iostream.c +++ b/src/flisp/iostream.c @@ -334,7 +334,7 @@ value_t fl_ioreaduntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_setbuf(&dest, data, 80, 0); char delim = get_delim_arg(fl_ctx, args[1], "io.readuntil"); ios_t *src = toiostream(fl_ctx, args[0], "io.readuntil"); - size_t n = ios_copyuntil(&dest, src, delim); + size_t n = ios_copyuntil(&dest, src, delim, 0); cv->len = n; if (dest.buf != data) { // outgrew initial space @@ -353,7 +353,7 @@ value_t fl_iocopyuntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_t *dest = toiostream(fl_ctx, args[0], "io.copyuntil"); ios_t *src = toiostream(fl_ctx, args[1], "io.copyuntil"); char delim = get_delim_arg(fl_ctx, args[2], "io.copyuntil"); - return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim)); + return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim, 0)); } value_t fl_iocopy(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) From 2214b93be533722aab8ec4881ff9de5067d4b196 Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Tue, 10 Jan 2017 14:54:43 +0200 Subject: [PATCH 08/15] Fix style according to comments --- NEWS.md | 2 +- base/io.jl | 1 - src/support/ios.c | 21 +++++++++++---------- src/support/ios.h | 2 +- src/sys.c | 5 ++++- test/iobuffer.jl | 5 +++++ test/spawn.jl | 2 +- 7 files changed, 23 insertions(+), 15 deletions(-) diff --git a/NEWS.md b/NEWS.md index 483e3cce24953..af597e69cf39b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -165,7 +165,7 @@ Compiler/Runtime improvements Deprecated or removed --------------------- - * One argument methods to `readline`, `readlines` and `eachline` have been depracated in + * One argument methods to `readline`, `readlines` and `eachline` have been deprecated in order to add new `chomp::Bool` argument which will default to `true` in future versions ([#19944]). * `isdefined(a::Array, i::Int)` has been deprecated in favor of `isassigned` ([#18346]). diff --git a/base/io.jl b/base/io.jl index 19dc69bba57cf..83aaa4a87b8ff 100644 --- a/base/io.jl +++ b/base/io.jl @@ -558,7 +558,6 @@ otherwise newline characters(s) are stripped from result. """ eachline(stream::IO, chomp::Bool=true) = EachLine(stream, chomp) - function eachline(filename::AbstractString, chomp::Bool=true) s = open(filename) EachLine(s, chomp, ()->close(s)) diff --git a/src/support/ios.c b/src/support/ios.c index 6a3cc11825fc9..acfdf21f5ccd7 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -821,7 +821,10 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp) } else { size_t ntowrite = pd - (from->buf+from->bpos) + 1; - size_t nchomp = ios_nchomp(from, ntowrite, chomp); + size_t nchomp = 0; + if (chomp) { + nchomp = ios_nchomp(from, ntowrite); + } written = ios_write(to, from->buf+from->bpos, ntowrite - nchomp); from->bpos += ntowrite; total += written; @@ -832,16 +835,14 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp) return total; } -size_t ios_nchomp(ios_t *from, size_t ntowrite, uint8_t chomp) +size_t ios_nchomp(ios_t *from, size_t ntowrite) { - size_t nchomp = 0; - if (chomp) { - if (ntowrite > 1 && from->buf[from->bpos+ntowrite - 2] == '\r') { - nchomp = 2; - } - else { - nchomp = 1; - } + size_t nchomp; + if (ntowrite > 1 && from->buf[from->bpos+ntowrite - 2] == '\r') { + nchomp = 2; + } + else { + nchomp = 1; } return nchomp; } diff --git a/src/support/ios.h b/src/support/ios.h index f901e74f21893..02480237a3389 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -95,7 +95,7 @@ JL_DLLEXPORT void ios_set_readonly(ios_t *s); JL_DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes); JL_DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from); JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp); -JL_DLLEXPORT size_t ios_nchomp(ios_t *from, size_t ntowrite, uint8_t chomp); +JL_DLLEXPORT size_t ios_nchomp(ios_t *from, size_t ntowrite); // ensure at least n bytes are buffered if possible. returns # available. JL_DLLEXPORT size_t ios_readprep(ios_t *from, size_t n); diff --git a/src/sys.c b/src/sys.c index 49af7c42c925d..3ced15cd45c73 100644 --- a/src/sys.c +++ b/src/sys.c @@ -258,7 +258,10 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint if (pd) { size_t n = pd-(s->buf+s->bpos)+1; if (str) { - size_t nchomp = ios_nchomp(s, n, chomp); + size_t nchomp = 0; + if (chomp) { + nchomp = ios_nchomp(s, n); + } jl_value_t *str = jl_pchar_to_string(s->buf + s->bpos, n - nchomp); s->bpos += n; return str; diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 459a76bd57112..6e53312b1f39c 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -80,6 +80,11 @@ write(io,"\n\r\n\n\r \n") > 0 @test readlines(io, false) == String["\n", "\r\n", "\n", "\r \n"] write(io,"\n\r\n\n\r \n") > 0 @test readlines(io, true) == String["", "", "", "\r "] +@test write(io,"α\nβ\nγ\nδ") > 0 +readlines(io, true) == String["α", "β", "γ", "δ"] +@test write(io,"α\nβ\nγ\nδ") > 0 +readlines(io, false) == String["α", "β", "γ", "δ"] +String["α\n","β\n","γ\n","δ"] Base.compact(io) @test position(io) == 0 @test ioslength(io) == 0 diff --git a/test/spawn.jl b/test/spawn.jl index e89fbcc82c310..f94e0b5e1c916 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -353,7 +353,7 @@ let fname = tempname() cmd = pipeline(`busybox echo asdf`,`busybox cat`) end end - for line in eachline(STDIN) + for line in eachline(STDIN, false) run(cmd) end """ From c6673dc99d54576c4c7239a9db7b6899e6174c29 Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Tue, 10 Jan 2017 16:24:09 +0200 Subject: [PATCH 09/15] Fix deprecation in pkg --- base/deprecated.jl | 4 ++-- base/pkg/reqs.jl | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index c2ab600b80071..9c058342b02a0 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1547,8 +1547,8 @@ eval(LibGit2, :(Base.@deprecate_binding GitAnyObject GitUnknownObject)) @deprecate readlines(s::IO) readlines(s, false) @deprecate readlines(s::IOStream) readlines(s, false) @deprecate readlines(filename::AbstractString) readlines(filename, false) -@deprecate eachline(s::IO) readlines(s, false) -@deprecate eachline(s::IOStream) readlines(s, false) +@deprecate eachline(s::IO) eachline(s, false) +@deprecate eachline(s::IOStream) eachline(s, false) @deprecate eachline(filename::AbstractString) eachline(filename, false) # End deprecations scheduled for 0.6 diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index 461e15b0abaf6..e647ef18caaad 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -65,8 +65,7 @@ end function read(readable::Union{IO,Base.AbstractCmd}) lines = Line[] - for line in eachline(readable) - line = chomp(line) + for line in eachline(readable, true) push!(lines, ismatch(r"^\s*(?:#|$)", line) ? Comment(line) : Requirement(line)) end return lines From cd48d62ed9cc83938bce957d405c4a5ebe64430d Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Tue, 10 Jan 2017 17:10:09 +0200 Subject: [PATCH 10/15] Simplify chomp --- base/io.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/base/io.jl b/base/io.jl index 83aaa4a87b8ff..d19f81da8e4ba 100644 --- a/base/io.jl +++ b/base/io.jl @@ -186,11 +186,9 @@ end readline() = readline(STDIN, false) function readline(s::IO, chomp::Bool=true) - chomp || return String(readuntil(s, 0x0a)) - line = readuntil(s, 0x0a) i = length(line) - if i == 0 || line[i] != 0x0a + if !chomp || i == 0 || line[i] != 0x0a return String(line) elseif i < 2 || line[i-1] != 0x0d return String(resize!(line,i-1)) From b8ab89a28211ee4c5d6d3f2c488c181cc61746cb Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Tue, 10 Jan 2017 18:51:55 +0200 Subject: [PATCH 11/15] Remove default value from chomp --- base/io.jl | 25 +++++++++++++------------ base/iostream.jl | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/base/io.jl b/base/io.jl index d19f81da8e4ba..30026c89cb8f3 100644 --- a/base/io.jl +++ b/base/io.jl @@ -167,8 +167,9 @@ The text is assumed to be encoded in UTF-8. readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), filename) """ - readline(stream::IO=STDIN, chomp::Bool=true) - readline(filename::AbstractString, chomp::Bool=true) + readline() + readline(stream, chomp::Bool) + readline(filename::AbstractString, chomp::Bool) Read a single line of text from the given I/O stream or file (defaults to `STDIN`). Lines in the input can end in `'\\n'` or `"\\r\\n"`. When reading from a file, the text is @@ -178,14 +179,14 @@ If `chomp=false` trailing newline character(s) will be included in the output (if reached before the end of the input); otherwise newline characters(s) are stripped from result. """ -function readline(filename::AbstractString, chomp::Bool=true) +function readline(filename::AbstractString, chomp::Bool) open(filename) do f readline(f, chomp) end end readline() = readline(STDIN, false) -function readline(s::IO, chomp::Bool=true) +function readline(s::IO, chomp::Bool) line = readuntil(s, 0x0a) i = length(line) if !chomp || i == 0 || line[i] != 0x0a @@ -198,8 +199,8 @@ function readline(s::IO, chomp::Bool=true) end """ - readlines(stream::IO, chomp::Bool=true) - readlines(filename::AbstractString, chomp::Bool=true) + readlines(stream::IO, chomp::Bool) + readlines(filename::AbstractString, chomp::Bool) Read all lines of an I/O stream or a file as a vector of strings. Lines in the input can end in `'\\n'` or `"\\r\\n"`. @@ -208,7 +209,7 @@ The text is assumed to be encoded in UTF-8. If `chomp=false` trailing newline character(s) will be included in the output; otherwise newline characters(s) are stripped from result. """ -function readlines(filename::AbstractString, chomp::Bool=true) +function readlines(filename::AbstractString, chomp::Bool) open(filename) do f readlines(f, chomp) end @@ -544,8 +545,8 @@ type EachLine end """ - eachline(stream::IO, chomp::Bool=true) - eachline(filename::AbstractString, chomp::Bool=true) + eachline(stream::IO, chomp::Bool) + eachline(filename::AbstractString, chomp::Bool) Create an iterable object that will yield each line from an I/O stream or a file. Lines in the input can end in `'\\n'` or `"\\r\\n"`. @@ -554,9 +555,9 @@ The text is assumed to be encoded in UTF-8. If `chomp=false` trailing newline character(s) will be included in the output; otherwise newline characters(s) are stripped from result. """ -eachline(stream::IO, chomp::Bool=true) = EachLine(stream, chomp) +eachline(stream::IO, chomp::Bool) = EachLine(stream, chomp) -function eachline(filename::AbstractString, chomp::Bool=true) +function eachline(filename::AbstractString, chomp::Bool) s = open(filename) EachLine(s, chomp, ()->close(s)) end @@ -573,7 +574,7 @@ end next(itr::EachLine, nada) = (readline(itr.stream, itr.chomp), nothing) eltype(::Type{EachLine}) = String -readlines(s::IO=STDIN, chomp::Bool=true) = collect(eachline(s, chomp)) +readlines(s::IO, chomp::Bool) = collect(eachline(s, chomp)) iteratorsize(::Type{EachLine}) = SizeUnknown() diff --git a/base/iostream.jl b/base/iostream.jl index 151ac63ab7f93..9ea8e382703d9 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -225,7 +225,7 @@ function readuntil(s::IOStream, delim::UInt8) ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 0, 0) end -function readline(s::IOStream, chomp::Bool=false) +function readline(s::IOStream, chomp::Bool) ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, '\n', 1, chomp) end From aafa71a57aae0c7d651e552362c1374d7055f473 Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Tue, 10 Jan 2017 19:51:08 +0200 Subject: [PATCH 12/15] More iobuffer tests --- base/deprecated.jl | 1 + test/iobuffer.jl | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 9c058342b02a0..ef4102075bd78 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1541,6 +1541,7 @@ unsafe_wrap(::Type{String}, p::Cstring, len::Integer, own::Bool=false) = eval(LibGit2, :(Base.@deprecate_binding GitAnyObject GitUnknownObject)) # Deprecate one argument method in order to make chomp=true the new default #19944 +# NOTE remember to add the chomp=true default to function definitions when removing these @deprecate readline(s::IO) readline(s, false) @deprecate readline(s::IOStream) readline(s, false) @deprecate readline(filename::AbstractString) readline(filename, false) diff --git a/test/iobuffer.jl b/test/iobuffer.jl index 6e53312b1f39c..4d3999bc2a557 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -81,10 +81,13 @@ write(io,"\n\r\n\n\r \n") > 0 write(io,"\n\r\n\n\r \n") > 0 @test readlines(io, true) == String["", "", "", "\r "] @test write(io,"α\nβ\nγ\nδ") > 0 -readlines(io, true) == String["α", "β", "γ", "δ"] +@test readlines(io, true) == String["α", "β", "γ", "δ"] @test write(io,"α\nβ\nγ\nδ") > 0 -readlines(io, false) == String["α", "β", "γ", "δ"] -String["α\n","β\n","γ\n","δ"] +@test readlines(io, false) == String["α\n","β\n","γ\n","δ"] +@test readlines(IOBuffer(""), false) == [] +@test readlines(IOBuffer(""), true) == [] +@test readlines(IOBuffer("first\nsecond"), false) == String["first\n", "second"] +@test readlines(IOBuffer("first\nsecond"), true) == String["first", "second"] Base.compact(io) @test position(io) == 0 @test ioslength(io) == 0 From b5d9a9890efdc5889652741817bb12eb73b0dfdb Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Thu, 12 Jan 2017 20:47:34 +0200 Subject: [PATCH 13/15] Make chomp=true default, remove deprecations --- NEWS.md | 6 +++--- base/deprecated.jl | 12 ------------ base/io.jl | 24 ++++++++++++------------ base/iostream.jl | 2 +- src/support/ios.c | 1 + 5 files changed, 17 insertions(+), 28 deletions(-) diff --git a/NEWS.md b/NEWS.md index af597e69cf39b..5fc9d0048914b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -33,6 +33,9 @@ Breaking changes This section lists changes that do not have deprecation warnings. + * `readline`, `readlines` and `eachline` return lines without line ends by default. + You can use `readline(s, false`) to get the old behavior and include EOL character(s). ([#19944]). + * `String`s no longer have a `.data` field (as part of a significant performance improvement). Use `Vector{UInt8}(str)` to access a string as a byte array. However, allocating the `Vector` object has overhead. You can also use @@ -165,9 +168,6 @@ Compiler/Runtime improvements Deprecated or removed --------------------- - * One argument methods to `readline`, `readlines` and `eachline` have been deprecated in - order to add new `chomp::Bool` argument which will default to `true` in future versions ([#19944]). - * `isdefined(a::Array, i::Int)` has been deprecated in favor of `isassigned` ([#18346]). * `is` has been deprecated in favor of `===` (which used to be an alias for `is`) ([#17758]). diff --git a/base/deprecated.jl b/base/deprecated.jl index ef4102075bd78..773ec6ae95bfb 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1540,16 +1540,4 @@ unsafe_wrap(::Type{String}, p::Cstring, len::Integer, own::Bool=false) = # Rename LibGit2.GitAnyObject to LibGit2.GitUnknownObject (part of #19839) eval(LibGit2, :(Base.@deprecate_binding GitAnyObject GitUnknownObject)) -# Deprecate one argument method in order to make chomp=true the new default #19944 -# NOTE remember to add the chomp=true default to function definitions when removing these -@deprecate readline(s::IO) readline(s, false) -@deprecate readline(s::IOStream) readline(s, false) -@deprecate readline(filename::AbstractString) readline(filename, false) -@deprecate readlines(s::IO) readlines(s, false) -@deprecate readlines(s::IOStream) readlines(s, false) -@deprecate readlines(filename::AbstractString) readlines(filename, false) -@deprecate eachline(s::IO) eachline(s, false) -@deprecate eachline(s::IOStream) eachline(s, false) -@deprecate eachline(filename::AbstractString) eachline(filename, false) - # End deprecations scheduled for 0.6 diff --git a/base/io.jl b/base/io.jl index 30026c89cb8f3..c2f0c86c8d74b 100644 --- a/base/io.jl +++ b/base/io.jl @@ -168,8 +168,8 @@ readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), """ readline() - readline(stream, chomp::Bool) - readline(filename::AbstractString, chomp::Bool) + readline(stream, chomp::Bool=true) + readline(filename::AbstractString, chomp::Bool=true) Read a single line of text from the given I/O stream or file (defaults to `STDIN`). Lines in the input can end in `'\\n'` or `"\\r\\n"`. When reading from a file, the text is @@ -179,14 +179,14 @@ If `chomp=false` trailing newline character(s) will be included in the output (if reached before the end of the input); otherwise newline characters(s) are stripped from result. """ -function readline(filename::AbstractString, chomp::Bool) +function readline(filename::AbstractString, chomp::Bool=true) open(filename) do f readline(f, chomp) end end readline() = readline(STDIN, false) -function readline(s::IO, chomp::Bool) +function readline(s::IO, chomp::Bool=true) line = readuntil(s, 0x0a) i = length(line) if !chomp || i == 0 || line[i] != 0x0a @@ -199,8 +199,8 @@ function readline(s::IO, chomp::Bool) end """ - readlines(stream::IO, chomp::Bool) - readlines(filename::AbstractString, chomp::Bool) + readlines(stream::IO, chomp::Bool=true) + readlines(filename::AbstractString, chomp::Bool=true) Read all lines of an I/O stream or a file as a vector of strings. Lines in the input can end in `'\\n'` or `"\\r\\n"`. @@ -209,7 +209,7 @@ The text is assumed to be encoded in UTF-8. If `chomp=false` trailing newline character(s) will be included in the output; otherwise newline characters(s) are stripped from result. """ -function readlines(filename::AbstractString, chomp::Bool) +function readlines(filename::AbstractString, chomp::Bool=true) open(filename) do f readlines(f, chomp) end @@ -545,8 +545,8 @@ type EachLine end """ - eachline(stream::IO, chomp::Bool) - eachline(filename::AbstractString, chomp::Bool) + eachline(stream::IO, chomp::Bool=true) + eachline(filename::AbstractString, chomp::Bool=true) Create an iterable object that will yield each line from an I/O stream or a file. Lines in the input can end in `'\\n'` or `"\\r\\n"`. @@ -555,9 +555,9 @@ The text is assumed to be encoded in UTF-8. If `chomp=false` trailing newline character(s) will be included in the output; otherwise newline characters(s) are stripped from result. """ -eachline(stream::IO, chomp::Bool) = EachLine(stream, chomp) +eachline(stream::IO, chomp::Bool=true) = EachLine(stream, chomp) -function eachline(filename::AbstractString, chomp::Bool) +function eachline(filename::AbstractString, chomp::Bool=true) s = open(filename) EachLine(s, chomp, ()->close(s)) end @@ -574,7 +574,7 @@ end next(itr::EachLine, nada) = (readline(itr.stream, itr.chomp), nothing) eltype(::Type{EachLine}) = String -readlines(s::IO, chomp::Bool) = collect(eachline(s, chomp)) +readlines(s::IO, chomp::Bool=true) = collect(eachline(s, chomp)) iteratorsize(::Type{EachLine}) = SizeUnknown() diff --git a/base/iostream.jl b/base/iostream.jl index 9ea8e382703d9..11bffc79618db 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -225,7 +225,7 @@ function readuntil(s::IOStream, delim::UInt8) ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 0, 0) end -function readline(s::IOStream, chomp::Bool) +function readline(s::IOStream, chomp::Bool=true) ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, '\n', 1, chomp) end diff --git a/src/support/ios.c b/src/support/ios.c index acfdf21f5ccd7..32cba8fceb618 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -837,6 +837,7 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp) size_t ios_nchomp(ios_t *from, size_t ntowrite) { + assert(ntowrite > 0); size_t nchomp; if (ntowrite > 1 && from->buf[from->bpos+ntowrite - 2] == '\r') { nchomp = 2; From 1c349bb844679b91bb7825ee2f8849ae9729ebe7 Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Fri, 13 Jan 2017 16:55:54 +0200 Subject: [PATCH 14/15] Update news --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 5fc9d0048914b..12958e5ce10a1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -33,8 +33,8 @@ Breaking changes This section lists changes that do not have deprecation warnings. - * `readline`, `readlines` and `eachline` return lines without line ends by default. - You can use `readline(s, false`) to get the old behavior and include EOL character(s). ([#19944]). + * `readline`, `readlines` and `eachline` return lines without line endings by default. + You can use `readline(s, false)` to get the old behavior and include EOL character(s). ([#19944]). * `String`s no longer have a `.data` field (as part of a significant performance improvement). Use `Vector{UInt8}(str)` to access a string as a byte array. From 354ada4e8214a93aa594a5ae2059317c92389592 Mon Sep 17 00:00:00 2001 From: Matti Pastell <matti.pastell@helsinki.fi> Date: Sun, 15 Jan 2017 11:32:02 +0200 Subject: [PATCH 15/15] Remove true from `readline` calls --- base/libgit2/callbacks.jl | 2 +- base/libgit2/utils.jl | 2 +- base/markdown/GitHub/table.jl | 2 +- base/markdown/parse/util.jl | 4 ++-- base/pkg/dir.jl | 2 +- base/pkg/reqs.jl | 2 +- test/read.jl | 8 ++++---- test/spawn.jl | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 52da4edb28310..40ea56cb533a8 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -99,7 +99,7 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, else # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" open(privatekey) do f - passphrase_required = (readline(f, false); readline(f, true) == "Proc-Type: 4,ENCRYPTED") + passphrase_required = (readline(f, false); readline(f) == "Proc-Type: 4,ENCRYPTED") end end diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl index a0e4649b38348..530e515af1d20 100644 --- a/base/libgit2/utils.jl +++ b/base/libgit2/utils.jl @@ -31,7 +31,7 @@ function prompt(msg::AbstractString; default::AbstractString="", password::Bool= Base.getpass(msg) else print(msg) - readline(STDIN, true) + readline(STDIN) end isempty(uinput) ? default : uinput end diff --git a/base/markdown/GitHub/table.jl b/base/markdown/GitHub/table.jl index 32d30ed98cf42..04485669dc936 100644 --- a/base/markdown/GitHub/table.jl +++ b/base/markdown/GitHub/table.jl @@ -7,7 +7,7 @@ end function parserow(stream::IO) withstream(stream) do - line = readline(stream, true) + line = readline(stream) row = split(line, r"(?<!\\)\|") length(row) == 1 && return row[1] == "" && shift!(row) diff --git a/base/markdown/parse/util.jl b/base/markdown/parse/util.jl index 8bbec8ce9d0b7..f36ca831d9b5f 100644 --- a/base/markdown/parse/util.jl +++ b/base/markdown/parse/util.jl @@ -46,7 +46,7 @@ function linecontains(io::IO, chars; allow_whitespace = true, eat = true, allowempty = false) start = position(io) - l = readline(io, true) + l = readline(io) length(l) == 0 && return allowempty result = allowempty @@ -99,7 +99,7 @@ function startswith(stream::IO, r::Regex; eat = true, padding = false) @assert Base.startswith(r.pattern, "^") start = position(stream) padding && skipwhitespace(stream) - line = readline(stream, true) + line = readline(stream) seek(stream, start) m = match(r, line) m === nothing && return "" diff --git a/base/pkg/dir.jl b/base/pkg/dir.jl index fe230e884fec3..fdb2231de3e24 100644 --- a/base/pkg/dir.jl +++ b/base/pkg/dir.jl @@ -67,7 +67,7 @@ end function getmetabranch() try - readline(joinpath(path(),"META_BRANCH"), true) + readline(joinpath(path(),"META_BRANCH")) catch err META_BRANCH end diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index e647ef18caaad..003b82496e3c9 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -65,7 +65,7 @@ end function read(readable::Union{IO,Base.AbstractCmd}) lines = Line[] - for line in eachline(readable, true) + for line in eachline(readable) push!(lines, ismatch(r"^\s*(?:#|$)", line) ? Comment(line) : Requirement(line)) end return lines diff --git a/test/read.jl b/test/read.jl index bcd3435682b4e..fdbd66b1bce7d 100644 --- a/test/read.jl +++ b/test/read.jl @@ -247,12 +247,12 @@ for (name, f) in l verbose && println("$name readlines...") @test readlines(io(), false) == readlines(IOBuffer(text), false) @test readlines(io(), false) == readlines(filename, false) - @test readlines(io(), true) == readlines(IOBuffer(text), true) - @test readlines(io(), true) == readlines(filename, true) + @test readlines(io()) == readlines(IOBuffer(text)) + @test readlines(io()) == readlines(filename) @test collect(eachline(io(), false)) == collect(eachline(IOBuffer(text), false)) @test collect(eachline(io(), false)) == collect(eachline(filename, false)) - @test collect(eachline(io(), true)) == collect(eachline(IOBuffer(text), true)) - @test collect(eachline(io(), true)) == collect(eachline(filename, true)) + @test collect(eachline(io())) == collect(eachline(IOBuffer(text))) + @test collect(eachline(io())) == collect(eachline(filename)) cleanup() diff --git a/test/spawn.jl b/test/spawn.jl index f94e0b5e1c916..fe97f648f1672 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -193,7 +193,7 @@ let r, t, sock @test !ismarked(sock) @test_throws ArgumentError reset(sock) @test !unmark(sock) - @test readline(sock, true) == "Goodbye, world..." + @test readline(sock) == "Goodbye, world..." #@test eof(sock) ## doesn't work close(sock) @test wait(t)