Merge csv-3.2.3

test/csv/parse/test_general.rb

@@ -199,6 +199,32 @@ line,5,jkl
                              field_size_limit: 2048 )
   end
 
+  def test_field_size_limit_max_allowed
+    column = "abcde"
+    assert_equal([[column]],
+                 CSV.parse("\"#{column}\"",
+                           field_size_limit: column.size + 1))
+  end
+
+  def test_field_size_limit_quote_simple
+    column = "abcde"
+    assert_parse_errors_out("\"#{column}\"",
+                            field_size_limit: column.size)
+  end
+
+  def test_field_size_limit_no_quote_implicitly
+    column = "abcde"
+    assert_parse_errors_out("#{column}",
+                            field_size_limit: column.size)
+  end
+
+  def test_field_size_limit_no_quote_explicitly
+    column = "abcde"
+    assert_parse_errors_out("#{column}",
+                            field_size_limit: column.size,
+                            quote_char: nil)
+  end
+
   def test_field_size_limit_in_extended_column_not_exceeding
     data = <<~DATA
       "a","b"
@@ -221,6 +247,59 @@ line,5,jkl
     assert_parse_errors_out(data, field_size_limit: 5)
   end
 
+  def test_max_field_size_controls_lookahead
+    assert_parse_errors_out( 'valid,fields,"' + BIG_DATA + '"',
+                             max_field_size: 2048 )
+  end
+
+  def test_max_field_size_max_allowed
+    column = "abcde"
+    assert_equal([[column]],
+                 CSV.parse("\"#{column}\"",
+                           max_field_size: column.size))
+  end
+
+  def test_max_field_size_quote_simple
+    column = "abcde"
+    assert_parse_errors_out("\"#{column}\"",
+                            max_field_size: column.size - 1)
+  end
+
+  def test_max_field_size_no_quote_implicitly
+    column = "abcde"
+    assert_parse_errors_out("#{column}",
+                            max_field_size: column.size - 1)
+  end
+
+  def test_max_field_size_no_quote_explicitly
+    column = "abcde"
+    assert_parse_errors_out("#{column}",
+                            max_field_size: column.size - 1,
+                            quote_char: nil)
+  end
+
+  def test_max_field_size_in_extended_column_not_exceeding
+    data = <<~DATA
+      "a","b"
+      "
+      2
+      ",""
+    DATA
+    assert_nothing_raised(CSV::MalformedCSVError) do
+      CSV.parse(data, max_field_size: 3)
+    end
+  end
+
+  def test_max_field_size_in_extended_column_exceeding
+    data = <<~DATA
+      "a","b"
+      "
+      2345
+      ",""
+    DATA
+    assert_parse_errors_out(data, max_field_size: 4)
+  end
+
   def test_row_sep_auto_cr
     assert_equal([["a"]], CSV.parse("a\r"))
   end
@@ -246,11 +325,7 @@ line,5,jkl
   private
   def assert_parse_errors_out(data, **options)
     assert_raise(CSV::MalformedCSVError) do
-      timeout = 0.2
-      if defined?(RubyVM::MJIT.enabled?) and RubyVM::MJIT.enabled?
-        timeout = 5  # for --jit-wait
-      end
-      Timeout.timeout(timeout) do
+      Timeout.timeout(0.2) do
         CSV.parse(data, **options)
         fail("Parse didn't error out")
       end

test/csv/parse/test_header.rb

@@ -218,6 +218,13 @@ A,B,C
     assert_equal([:one, :two_three], csv.headers)
   end
 
+  def test_builtin_symbol_raw_converter
+    csv = CSV.parse( "a b,c d", headers:           true,
+                     return_headers:    true,
+                     header_converters: :symbol_raw )
+    assert_equal([:"a b", :"c d"], csv.headers)
+  end
+
   def test_builtin_symbol_converter_with_punctuation
     csv = CSV.parse( "One, Two & Three ($)", headers:           true,
                                              return_headers:    true,
@@ -228,7 +235,7 @@ A,B,C
   def test_builtin_converters_with_blank_header
     csv = CSV.parse( "one,,three", headers:           true,
                                    return_headers:    true,
-                                   header_converters: [:downcase, :symbol] )
+                                   header_converters: [:downcase, :symbol, :symbol_raw] )
     assert_equal([:one, nil, :three], csv.headers)
   end
 

test/csv/parse/test_inputs_scanner.rb

@@ -0,0 +1,63 @@
+require_relative "../helper"
+
+class TestCSVParseInputsScanner < Test::Unit::TestCase
+  include Helper
+
+  def test_scan_keep_over_chunks_nested_back
+    input = CSV::Parser::UnoptimizedStringIO.new("abcdefghijklmnl")
+    scanner = CSV::Parser::InputsScanner.new([input],
+                                             Encoding::UTF_8,
+                                             nil,
+                                             chunk_size: 2)
+    scanner.keep_start
+    assert_equal("abc", scanner.scan_all(/[a-c]+/))
+    scanner.keep_start
+    assert_equal("def", scanner.scan_all(/[d-f]+/))
+    scanner.keep_back
+    scanner.keep_back
+    assert_equal("abcdefg", scanner.scan_all(/[a-g]+/))
+  end
+
+  def test_scan_keep_over_chunks_nested_drop_back
+    input = CSV::Parser::UnoptimizedStringIO.new("abcdefghijklmnl")
+    scanner = CSV::Parser::InputsScanner.new([input],
+                                             Encoding::UTF_8,
+                                             nil,
+                                             chunk_size: 3)
+    scanner.keep_start
+    assert_equal("ab", scanner.scan(/../))
+    scanner.keep_start
+    assert_equal("c", scanner.scan(/./))
+    assert_equal("d", scanner.scan(/./))
+    scanner.keep_drop
+    scanner.keep_back
+    assert_equal("abcdefg", scanner.scan_all(/[a-g]+/))
+  end
+
+  def test_each_line_keep_over_chunks_multibyte
+    input = CSV::Parser::UnoptimizedStringIO.new("ab\n\u{3000}a\n")
+    scanner = CSV::Parser::InputsScanner.new([input],
+                                             Encoding::UTF_8,
+                                             nil,
+                                             chunk_size: 1)
+    each_line = scanner.each_line("\n")
+    assert_equal("ab\n", each_line.next)
+    scanner.keep_start
+    assert_equal("\u{3000}a\n", each_line.next)
+    scanner.keep_back
+    assert_equal("\u{3000}a\n", scanner.scan_all(/[^,]+/))
+  end
+
+  def test_each_line_keep_over_chunks_fit_chunk_size
+    input = CSV::Parser::UnoptimizedStringIO.new("\na")
+    scanner = CSV::Parser::InputsScanner.new([input],
+                                             Encoding::UTF_8,
+                                             nil,
+                                             chunk_size: 1)
+    each_line = scanner.each_line("\n")
+    assert_equal("\n", each_line.next)
+    scanner.keep_start
+    assert_equal("a", each_line.next)
+    scanner.keep_back
+  end
+end

test/csv/parse/test_liberal_parsing.rb

@@ -28,6 +28,17 @@ class TestCSVParseLiberalParsing < Test::Unit::TestCase
                  CSV.parse_line(input, liberal_parsing: true))
   end
 
+  def test_endline_after_quoted_field_end
+    csv = CSV.new("A\r\n\"B\"\nC\r\n", liberal_parsing: true)
+    assert_equal(["A"], csv.gets)
+    error = assert_raise(CSV::MalformedCSVError) do
+      csv.gets
+    end
+    assert_equal('Illegal end-of-line sequence outside of a quoted field <"\n"> in line 2.',
+                 error.message)
+    assert_equal(["C"], csv.gets)
+  end
+
   def test_quote_after_column_separator
     error = assert_raise(CSV::MalformedCSVError) do
       CSV.parse_line('is,this "three," or four,fields', liberal_parsing: true)