diff --git a/src/FileFormats/NL/read.jl b/src/FileFormats/NL/read.jl index dc399d06ad..d1949e562e 100644 --- a/src/FileFormats/NL/read.jl +++ b/src/FileFormats/NL/read.jl @@ -5,6 +5,7 @@ # in the LICENSE.md file or at https://opensource.org/licenses/MIT. mutable struct _CacheModel + is_binary::Bool cache::Vector{UInt8} variable_type::Vector{_VariableType} variable_primal::Vector{Float64} @@ -17,6 +18,7 @@ mutable struct _CacheModel sense::MOI.OptimizationSense function _CacheModel() return new( + false, zeros(UInt8, 64), _VariableType[], Float64[], @@ -95,12 +97,18 @@ function _next_token(::Type{T}, io::IO, cache::Vector{UInt8}) where {T} end function _next(::Type{Float64}, io::IO, model::_CacheModel) + if model.is_binary + return read(io, Float64) + end nnz = _next_token(Float64, io, model.cache) @assert nnz > 0 return parse(Float64, String(model.cache[1:nnz])) end function _next(::Type{Int}, io::IO, model::_CacheModel) + if model.is_binary + return convert(Int, read(io, Int32)) + end nnz = _next_token(Int, io, model.cache) @assert nnz > 0 y = 0 @@ -112,13 +120,27 @@ function _next(::Type{Int}, io::IO, model::_CacheModel) return y end +function _next(::Type{Cchar}, io::IO, model::_CacheModel) + if model.is_binary + return read(io, Cchar) + end + byte = UInt8(' ') + while byte == UInt8(' ') + byte = read(io, UInt8) + end + return Cchar(byte) +end + """ - _read_til_newline(io::IO) + _read_til_newline(io::IO, model::_CacheModel) This function reads until it finds a new line character. This is useful for skipping comments. """ -function _read_til_newline(io::IO) +function _read_til_newline(io::IO, model::_CacheModel) + if model.is_binary + return + end while read(io, UInt8) != UInt8('\n') end return @@ -131,12 +153,12 @@ function _parse_expr(io::IO, model::_CacheModel) char = Char(read(io, UInt8)) if char == 'o' opcode = _next(Int, io, model) - _read_til_newline(io) + _read_til_newline(io, model) arity, op_func = _AMPL_TO_JULIA[opcode] op_sym = Symbol(op_func) if arity == -1 arity = _next(Int, io, model) - _read_til_newline(io) + _read_til_newline(io, model) if op_sym == :sum op_sym = :+ elseif op_sym == :minimum @@ -153,12 +175,12 @@ function _parse_expr(io::IO, model::_CacheModel) return parent elseif char == 'v' index = _next(Int, io, model) - _read_til_newline(io) + _read_til_newline(io, model) return MOI.VariableIndex(index + 1) else @assert char == 'n' ret = _next(Float64, io, model) - _read_til_newline(io) + _read_til_newline(io, model) return ret end end @@ -270,13 +292,16 @@ function _parse_header(io::IO, model::_CacheModel) # Line 1 # We don't support the binary format. byte = read(io, UInt8) - if byte != UInt8('g') + is_binary = false + if byte == UInt8('b') + is_binary = true + elseif byte != UInt8('g') error("Unable to parse NL file : unsupported mode $(Char(byte))") end # L1 has some magic bytes for AMPL internals (to quote David, "The numbers # on the first line matter to AMPL; for other uses, it is best simply to # supply the ones shown above.") - _read_til_newline(io) + _read_til_newline(io, model) # Line 2 # The number of variables n_var = _next(Int, io, model) @@ -292,19 +317,19 @@ function _parse_header(io::IO, model::_CacheModel) # The number of logical constraints. This one is optional, so just read til # the end of the line. # @assert _next(Int, io, model) == 0 - _read_til_newline(io) + _read_til_newline(io, model) # Line 3 # The number of nonlinear constraints @assert _next(Int, io, model) >= 0 # The number of nonlinear objectives @assert 0 <= _next(Int, io, model) <= 1 - _read_til_newline(io) + _read_til_newline(io, model) # Line 4 # The number of nonlinear network constraints @assert _next(Int, io, model) == 0 # The number of linear network constraints @assert _next(Int, io, model) == 0 - _read_til_newline(io) + _read_til_newline(io, model) # Line 5 # The number of nonlienar variables in constraints nlvc = _next(Int, io, model) @@ -312,7 +337,7 @@ function _parse_header(io::IO, model::_CacheModel) nlvo = _next(Int, io, model) # The number of nonlienar variables in constraints and objectives (both) nlvb = _next(Int, io, model) - _read_til_newline(io) + _read_til_newline(io, model) # Line 6 # The number of linear network variables @assert _next(Int, io, model) == 0 @@ -320,11 +345,11 @@ function _parse_header(io::IO, model::_CacheModel) @assert _next(Int, io, model) == 0 # The number of "arith" # TODO(odow): I don't know what this is. - @assert _next(Int, io, model) == 0 + _next(Int, io, model) # The "flags" entry. This is mainly used for specifying that we want duals. # Ignore when reading. _next(Int, io, model) - _read_til_newline(io) + _read_til_newline(io, model) # Line 7 # Number of binary variables nbv = _next(Int, io, model) @@ -336,17 +361,17 @@ function _parse_header(io::IO, model::_CacheModel) nlvci = _next(Int, io, model) # Number of integer variables in nonlinear objectives nlvoi = _next(Int, io, model) - _read_til_newline(io) + _read_til_newline(io, model) # Line 8 # Read the number of nonzeros in Jacobian and gradient, but don't do # anything with that information. @assert _next(Int, io, model) >= 0 @assert _next(Int, io, model) >= 0 - _read_til_newline(io) + _read_til_newline(io, model) # Line 9 # We don't support reading variable and constraint names, so just ignore # them - _read_til_newline(io) + _read_til_newline(io, model) # Line 10 # We don't support reading common subexpressions for _ in 1:5 @@ -354,7 +379,7 @@ function _parse_header(io::IO, model::_CacheModel) error("Unable to parse NL file : we don't support common exprs") end end - _read_til_newline(io) + _read_til_newline(io, model) # ========================================================================== # Deal with the integrality of variables. This is quite complicated, so go # read the README in this folder. @@ -387,6 +412,8 @@ function _parse_header(io::IO, model::_CacheModel) model.variable_type[offset] = types[i] end end + # Delay setting is_binary until the end of the header section + model.is_binary = is_binary return end @@ -412,10 +439,20 @@ end function _parse_section(io::IO, ::Val{'S'}, model::_CacheModel) k = _next(Int, io, model) n = _next(Int, io, model) - suffix = readline(io) - @warn("Skipping suffix: `S$k $n$suffix`") + suffix_name = if model.is_binary + len = _next(Int, io, model) + String(read(io, len)) + else + strip(readline(io)) + end + @warn("Skipping suffix: `S$k $n $suffix_name`") + # The “4” bit of k indicates whether the suffix is real (that is, double) + # valued or integer valued: (k&4) != 0 --> real valued. + T = ifelse(k & 4 != 0, Float64, Int) for _ in 1:n - _read_til_newline(io) + _ = _next(Int, io, model) + _ = _next(T, io, model) + _read_til_newline(io, model) end return end @@ -440,7 +477,7 @@ end function _parse_section(io::IO, ::Val{'C'}, model::_CacheModel) index = _next(Int, io, model) + 1 - _read_til_newline(io) + _read_til_newline(io, model) expr = _force_expr(_parse_expr(io, model)) current = model.constraints[index] if current == :() @@ -460,7 +497,7 @@ function _parse_section(io::IO, ::Val{'O'}, model::_CacheModel) @assert sense == 0 model.sense = MOI.MIN_SENSE end - _read_til_newline(io) + _read_til_newline(io, model) expr = _force_expr(_parse_expr(io, model)) if model.objective == :() model.objective = expr @@ -472,79 +509,83 @@ end function _parse_section(io::IO, ::Val{'x'}, model::_CacheModel) index = _next(Int, io, model) - _read_til_newline(io) + _read_til_newline(io, model) for _ in 1:index xi = _next(Int, io, model) + 1 v = _next(Float64, io, model) model.variable_primal[xi] = v - _read_til_newline(io) + _read_til_newline(io, model) end return end # TODO(odow): we don't read in dual starts. function _parse_section(io::IO, ::Val{'d'}, model::_CacheModel) - index = _next(Int, io, model) - _read_til_newline(io) - for _ in 1:index - _read_til_newline(io) + n = _next(Int, io, model) + _read_til_newline(io, model) + for _ in 1:n + _ = _next(Int, io, model) + _ = _next(Float64, io, model) + _read_til_newline(io, model) end return end function _parse_section(io::IO, ::Val{'r'}, model::_CacheModel) - _read_til_newline(io) + _read_til_newline(io, model) for i in 1:length(model.constraint_lower) - type = _next(Int, io, model) - if type == 0 + type = _next(Cchar, io, model) + if type == Cchar('0') model.constraint_lower[i] = _next(Float64, io, model) model.constraint_upper[i] = _next(Float64, io, model) - elseif type == 1 + elseif type == Cchar('1') model.constraint_upper[i] = _next(Float64, io, model) - elseif type == 2 + elseif type == Cchar('2') model.constraint_lower[i] = _next(Float64, io, model) - elseif type == 3 + elseif type == Cchar('3') # Free constraint else - @assert type == 4 + @assert type == Cchar('4') value = _next(Float64, io, model) model.constraint_lower[i] = value model.constraint_upper[i] = value end - _read_til_newline(io) + _read_til_newline(io, model) end return end function _parse_section(io::IO, ::Val{'b'}, model::_CacheModel) - _read_til_newline(io) + _read_til_newline(io, model) for i in 1:length(model.variable_lower) - type = _next(Int, io, model) - if type == 0 + type = _next(Cchar, io, model) + if type == Cchar('0') model.variable_lower[i] = _next(Float64, io, model) model.variable_upper[i] = _next(Float64, io, model) - elseif type == 1 + elseif type == Cchar('1') model.variable_upper[i] = _next(Float64, io, model) - elseif type == 2 + elseif type == Cchar('2') model.variable_lower[i] = _next(Float64, io, model) - elseif type == 3 + elseif type == Cchar('3') # Free variable else - @assert type == 4 + @assert type == Cchar('4') value = _next(Float64, io, model) model.variable_lower[i] = value model.variable_upper[i] = value end - _read_til_newline(io) + _read_til_newline(io, model) end return end # We ignore jacobian counts for now function _parse_section(io::IO, ::Val{'k'}, model::_CacheModel) - _read_til_newline(io) - for _ in 2:length(model.variable_lower) - _read_til_newline(io) + n = _next(Int, io, model) + _read_til_newline(io, model) + for _ in 1:n + _ = _next(Int, io, model) + _read_til_newline(io, model) end return end @@ -552,7 +593,7 @@ end function _parse_section(io::IO, ::Val{'J'}, model::_CacheModel) i = _next(Int, io, model) + 1 nnz = _next(Int, io, model) - _read_til_newline(io) + _read_til_newline(io, model) expr = Expr(:call, :+) for _ in 1:nnz x = _next(Int, io, model) @@ -560,7 +601,7 @@ function _parse_section(io::IO, ::Val{'J'}, model::_CacheModel) if !iszero(c) push!(expr.args, Expr(:call, :*, c, MOI.VariableIndex(x + 1))) end - _read_til_newline(io) + _read_til_newline(io, model) end if length(expr.args) == 1 # Linear part is just zeros @@ -575,7 +616,7 @@ end function _parse_section(io::IO, ::Val{'G'}, model::_CacheModel) i = _next(Int, io, model) + 1 nnz = _next(Int, io, model) - _read_til_newline(io) + _read_til_newline(io, model) expr = Expr(:call, :+) for _ in 1:nnz x = _next(Int, io, model) @@ -583,7 +624,7 @@ function _parse_section(io::IO, ::Val{'G'}, model::_CacheModel) if !iszero(c) push!(expr.args, Expr(:call, :*, c, MOI.VariableIndex(x + 1))) end - _read_til_newline(io) + _read_til_newline(io, model) end if length(expr.args) == 1 # Linear part is just zeros diff --git a/test/FileFormats/NL/read.jl b/test/FileFormats/NL/read.jl index eed46db26a..df656ad93e 100644 --- a/test/FileFormats/NL/read.jl +++ b/test/FileFormats/NL/read.jl @@ -103,14 +103,14 @@ function test_parse_expr_maximum() return end -function test_parse_header_binary() +function test_parse_header_unsupported_mode() model = NL._CacheModel() NL._resize_variables(model, 4) io = IOBuffer() - write(io, "b3 1 1 0\n") + write(io, "z3 1 1 0\n") seekstart(io) @test_throws( - ErrorException("Unable to parse NL file : unsupported mode b"), + ErrorException("Unable to parse NL file : unsupported mode z"), NL._parse_header(io, model), ) return @@ -147,7 +147,7 @@ function test_parse_header_assertion_errors() "g4 1 1 0\n3 3 1 0 0 0\n0 0\n0 1\n", "g3 1 1 0\n4 2 1 0 1 0\n2 1\n0 0\n4 0 0\n1 0 0 1\n", "g3 1 1 0\n4 2 1 0 1 0\n2 1\n0 0\n4 0 0\n0 1 0 1\n", - "g3 1 1 0\n4 2 1 0 1 0\n2 1\n0 0\n4 0 0\n0 0 1 1\n", + # "g3 1 1 0\n4 2 1 0 1 0\n2 1\n0 0\n4 0 0\n0 0 1 1\n", "g3 1 1 0\n4 2 1 0 1 0\n2 1\n0 0\n4 0 0\n0 0 0 1\n0 0 0 2 0\n-1 0\n", "g3 1 1 0\n4 2 1 0 1 0\n2 1\n0 0\n4 0 0\n0 0 0 1\n0 0 0 2 0\n0 -1\n", ] @@ -277,13 +277,13 @@ function test_parse_r() write( io, """ -r# can stick a comment anywhere -1 3.3 -3 -0 1.1 2.2 # can stick a comment anywhere -4 5.5 -2 4.4# can stick a comment anywhere -""", + r# can stick a comment anywhere + 1 3.3 + 3 + 0 1.1 2.2 # can stick a comment anywhere + 4 5.5 + 2 4.4# can stick a comment anywhere + """, ) seekstart(io) NL._parse_section(io, model) @@ -300,13 +300,13 @@ function test_parse_b() write( io, """ -b# can stick a comment anywhere -1 3.3 -3# can stick a comment anywhere -0 1.1 2.2 -4 5.5 -2 4.4 # can stick a comment anywhere -""", + b# can stick a comment anywhere + 1 3.3 + 3# can stick a comment anywhere + 0 1.1 2.2 + 4 5.5 + 2 4.4 # can stick a comment anywhere + """, ) seekstart(io) NL._parse_section(io, model) @@ -323,10 +323,10 @@ function test_parse_k() write( io, """ -k -2 # can stick a comment anywhere -4 -""", + k2 + 2 # can stick a comment anywhere + 4 + """, ) seekstart(io) NL._parse_section(io, model) @@ -573,14 +573,14 @@ function test_parse_S() io, """ S0 8 zork - 02 - 16 - 27 - 38 - 49 - 53 - 65 - 84 + 0 2 + 1 6 + 2 7 + 3 8 + 4 9 + 5 3 + 6 5 + 8 4 """, ) seekstart(io) @@ -592,6 +592,32 @@ function test_parse_S() return end +function test_parse_S_Float64() + model = NL._CacheModel() + io = IOBuffer() + write( + io, + """ + S4 8 zork + 0 2.0 + 1 6.0 + 2 7.0 + 3 8.0 + 4 9.0 + 5 3.0 + 6 5.0 + 8 4.0 + """, + ) + seekstart(io) + @test_logs( + (:warn, "Skipping suffix: `S4 8 zork`"), + NL._parse_section(io, model), + ) + @test eof(io) + return +end + function test_hs071() model = NL.Model() open(joinpath(@__DIR__, "data", "hs071.nl"), "r") do io @@ -813,6 +839,233 @@ function test_nl_read_scalar_affine_function() return end +function test_binary_next_Float64() + model = NL._CacheModel() + model.is_binary = true + sequence = [0.0, 1.0, -2.0, 1.234, 1e-12] + io = IOBuffer() + for s in sequence + write(io, s) + end + seekstart(io) + for s in sequence + @test NL._next(Float64, io, model) === s + end + return +end + +function test_binary_next_Int() + model = NL._CacheModel() + model.is_binary = true + sequence = Int32[0, 1, 2, -3, -5, typemax(Int32), typemin(Int32)] + io = IOBuffer() + for s in sequence + write(io, s) + end + seekstart(io) + for s in sequence + @test NL._next(Int, io, model) === Int(s) + end + return +end + +function test_binary_read_til_newline() + model = NL._CacheModel() + model.is_binary = true + io = IOBuffer() + @test NL._read_til_newline(io, model) === nothing + @test position(io) == 0 + return +end + +function test_parse_header_binary() + model = NL._CacheModel() + io = IOBuffer() + write( + io, + """b3 0 1 0 # problem test_simple + 2 1 1 0 0 # vars, constraints, objectives, ranges, eqns + 1 1 # nonlinear constraints, objectives + 0 0 # network constraints: nonlinear, linear + 1 2 0 # nonlinear vars in constraints, objectives, both + 0 0 1 1 # linear network variables; functions; arith, flags + 0 0 0 0 0 # discrete variables: binary, integer, nonlinear (b,c,o) + 1 1 # nonzeros in Jacobian, gradients + 0 0 # max name lengths: constraints, variables + 0 0 0 0 0 # common exprs: b,c,o,c1,o1 + """, + ) + seekstart(io) + NL._parse_header(io, model) + @test model.is_binary + return +end + +function test_binary_parse_O() + model = NL._CacheModel() + model.is_binary = true + NL._resize_variables(model, 4) + io = IOBuffer() + write(io, Cchar('O'), Int32(0), Int32(0)) + write(io, Cchar('o'), Int32(2)) + write(io, Cchar('v'), Int32(0)) + write(io, Cchar('o'), Int32(2)) + write(io, Cchar('v'), Int32(2)) + write(io, Cchar('o'), Int32(2)) + write(io, Cchar('v'), Int32(3)) + write(io, Cchar('v'), Int32(1)) + seekstart(io) + NL._parse_section(io, model) + @test eof(io) + @test model.sense == MOI.MIN_SENSE + x = MOI.VariableIndex.(1:4) + @test model.objective == :($(x[1]) * ($(x[3]) * ($(x[4]) * $(x[2])))) + return +end + +function test_binary_parse_O_max() + model = NL._CacheModel() + model.is_binary = true + NL._resize_variables(model, 4) + io = IOBuffer() + write(io, Cchar('O'), Int32(0), Int32(1), Cchar('v'), Int32(0)) + seekstart(io) + NL._parse_section(io, model) + @test eof(io) + @test model.sense == MOI.MAX_SENSE + x = MOI.VariableIndex(1) + @test model.objective == :(+$x) + return +end + +function test_binary_parse_x() + model = NL._CacheModel() + model.is_binary = true + NL._resize_variables(model, 5) + io = IOBuffer() + write(io, Cchar('x'), Int32(3)) + write(io, Int32(0), 1.1) + write(io, Int32(3), 2.2) + write(io, Int32(2), 3.3) + seekstart(io) + NL._parse_section(io, model) + @test eof(io) + @test model.variable_primal == [1.1, 0.0, 3.3, 2.2, 0.0] + return +end + +function test_parse_d() + model = NL._CacheModel() + model.is_binary = true + io = IOBuffer() + write(io, Cchar('d'), Int32(3)) + write(io, Int32(0), 1.1) + write(io, Int32(3), 2.2) + write(io, Int32(2), 3.3) + seekstart(io) + NL._parse_section(io, model) + @test eof(io) + return +end + +function test_binary_parse_r() + model = NL._CacheModel() + model.is_binary = true + NL._resize_constraints(model, 5) + io = IOBuffer() + write(io, Cchar('r')) + write(io, Cchar('1'), 3.3) + write(io, Cchar('3')) + write(io, Cchar('0'), 1.1, 2.2) + write(io, Cchar('4'), 5.5) + write(io, Cchar('2'), 4.4) + seekstart(io) + NL._parse_section(io, model) + @test eof(io) + @test model.constraint_lower == [-Inf, -Inf, 1.1, 5.5, 4.4] + @test model.constraint_upper == [3.3, Inf, 2.2, 5.5, Inf] + return +end + +function test_binary_parse_b() + model = NL._CacheModel() + model.is_binary = true + NL._resize_variables(model, 5) + io = IOBuffer() + write(io, Cchar('b')) + write(io, Cchar('1'), 3.3) + write(io, Cchar('3')) + write(io, Cchar('0'), 1.1, 2.2) + write(io, Cchar('4'), 5.5) + write(io, Cchar('2'), 4.4) + seekstart(io) + NL._parse_section(io, model) + @test eof(io) + @test model.variable_lower == [-Inf, -Inf, 1.1, 5.5, 4.4] + @test model.variable_upper == [3.3, Inf, 2.2, 5.5, Inf] + return +end + +function test_binary_parse_k() + model = NL._CacheModel() + model.is_binary = true + NL._resize_variables(model, 3) + io = IOBuffer() + write(io, Cchar('k'), Int32(2)) + write(io, Int32(2)) + write(io, Int32(4)) + seekstart(io) + NL._parse_section(io, model) + @test eof(io) + return +end + +function test_binary_parse_S() + model = NL._CacheModel() + model.is_binary = true + io = IOBuffer() + write(io, Cchar('S'), Int32(0), Int32(8)) + write(io, Int32(4), Cchar('z'), Cchar('o'), Cchar('r'), Cchar('k')) + write(io, Int32(0), Int32(2)) + write(io, Int32(1), Int32(6)) + write(io, Int32(2), Int32(7)) + write(io, Int32(3), Int32(8)) + write(io, Int32(4), Int32(9)) + write(io, Int32(5), Int32(3)) + write(io, Int32(6), Int32(5)) + write(io, Int32(8), Int32(4)) + seekstart(io) + @test_logs( + (:warn, "Skipping suffix: `S0 8 zork`"), + NL._parse_section(io, model), + ) + @test eof(io) + return +end + +function test_binary_parse_S_Float64() + model = NL._CacheModel() + model.is_binary = true + io = IOBuffer() + write(io, Cchar('S'), Int32(4), Int32(8)) + write(io, Int32(4), Cchar('z'), Cchar('o'), Cchar('r'), Cchar('k')) + write(io, Int32(0), 2.1) + write(io, Int32(1), 6.1) + write(io, Int32(2), 7.1) + write(io, Int32(3), 8.1) + write(io, Int32(4), 9.1) + write(io, Int32(5), 3.1) + write(io, Int32(6), 5.1) + write(io, Int32(8), 4.1) + seekstart(io) + @test_logs( + (:warn, "Skipping suffix: `S4 8 zork`"), + NL._parse_section(io, model), + ) + @test eof(io) + return +end + end TestNonlinearRead.runtests()