[GR-13303] [GR-9815] [GR-9816] [GR-10821] [GR-8904] Support the full FFI.

PullRequest: truffleruby/801

Author: eregon

.gitignore

@@ -44,6 +44,8 @@ src/main/c/truffleposix/truffleposix.o
 src/main/c/truffleposix/libtruffleposix.so
 src/main/c/truffleposix/libtruffleposix.dylib
 
+spec/ffi/.bundle/config
+
 spec/truffle/fixtures/libtestnfi.so
 spec/truffle/fixtures/libtestnfi.dylib
 

.rubocop.yml

@@ -7,7 +7,7 @@ AllCops:
     - 'lib/truffle/date/format.rb' # crashes Rubocop in VariableForce
     - 'lib/truffle/date/delta/parser.rb' # generated code
     - 'lib/truffle/socket/mri.rb' # taken from MRI
-    - 'lib/truffle/ffi/library.rb' # taken from FFI gem
+    - 'lib/truffle/ffi/**/*.rb' # taken from FFI gem
     - 'src/test/ruby/types.rb' # deliberately strange code for debugging purposes
     - 'src/test/ruby/lexical-context.rb' # deliberately strange code for debugging purposes
 

CHANGELOG.md

@@ -11,6 +11,7 @@ New features:
 
 * Rounding modes have been implemented or improved for `Float`, `Rational`, `BigDecimal` (#1509).
 * Support Homebrew installed in other prefixes than `/usr/local` (#1583).
+* Added a pure-Ruby implementation of FFI which passes almost all Ruby FFI specs (#1529, #1524).
 
 Changes:
 

ci.jsonnet

@@ -379,7 +379,7 @@ local part_definitions = {
         ["mx", "unittest", "org.truffleruby"],
         ["mx", "tck"],
       ] + jt(["test", "specs"] + self["$.run.specs"].test_spec_options),
-              # + jt(["test", "specs", ":next"]) disabled as it's currently empty and MSpec doesn't support empty sets of files
+      # + jt(["test", "specs", ":next"]) disabled as it's currently empty and MSpec doesn't support empty sets of files
     },
 
     test_fast: {
@@ -403,6 +403,7 @@ local part_definitions = {
     test_gems: { run+: jt(["test", "gems"]) },
     test_ecosystem: { run+: jt(["test", "ecosystem"]) },
     test_compiler: { run+: jt(["test", "compiler"]) },
+    test_ffi: { run+: [["test/truffle/ffi.sh"]] },
 
     test_cexts: {
       is_after+:: ["$.use.common"],
@@ -522,7 +523,7 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
     {
       local linux_gate = $.platform.linux + $.cap.gate + $.jdk.labsjdk8 + $.use.common + $.use.build + { timelimit: "01:00:00" },
 
-      "ruby-test-specs-linux": linux_gate + $.run.test_unit_tck_specs + $.run.test_basictest + { timelimit: "35:00" },
+      "ruby-test-specs-linux": linux_gate + $.run.test_unit_tck_specs + $.run.test_basictest + $.run.test_ffi + { timelimit: "35:00" },
       "ruby-test-fast-linux":  linux_gate + $.run.test_fast + { timelimit: "30:00" }, # To catch missing slow tags
       "ruby-test-mri-linux":   linux_gate + $.run.test_mri + { timelimit: "30:00" },
       "ruby-test-integration": linux_gate + $.run.test_integration,
@@ -539,7 +540,7 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
     {
       local darwin_gate = $.platform.darwin + $.cap.gate + $.jdk.labsjdk8 + $.use.common + $.use.build + { timelimit: "01:00:00" },
 
-      "ruby-test-specs-darwin": darwin_gate + $.run.test_unit_tck_specs + $.run.test_basictest + { timelimit: "45:00" },
+      "ruby-test-specs-darwin": darwin_gate + $.run.test_unit_tck_specs + $.run.test_basictest + $.run.test_ffi + { timelimit: "45:00" },
       "ruby-test-mri-darwin":   darwin_gate + $.run.test_mri,
       "ruby-test-cexts-darwin": darwin_gate + $.use.gem_test_pack + $.run.test_cexts,
     } +

doc/legal/legal.md

@@ -174,6 +174,11 @@ licenses. We've added the licenses to the individual files.
 We do not distribute MSpec or the Ruby Specs, but they are both copyright 2008
 Engine Yard and are released under an MIT licence (see `mit.txt`).
 
+## FFI Specs
+
+We do not distribute the FFI Specs, but they are copyright 2008-2014
+Ruby-FFI contributors and are released under an MIT licence (see `mit.txt`).
+
 ## Jay
 
 TruffleRuby uses the Jay parser generator, modified from

doc/user/compatibility.md

@@ -74,8 +74,8 @@ The following standard libraries are unsupported.
 but not enough to run anything serious.
 
 We provide our own included implementation of the interface of the `ffi` gem,
-like JRuby and Rubinius, but the implemention of this is limited at the
-moment.
+like JRuby and Rubinius. The implementation should be fairly complete and passes
+all the specs of the `ffi` gem except for some rarely-used corner cases.
 
 #### Safe levels
 

lib/truffle/ffi.rb

@@ -6,126 +6,42 @@
 # GNU General Public License version 2, or
 # GNU Lesser General Public License version 2.1.
 
-require_relative 'truffle/ffi/ffi'
-require_relative 'truffle/ffi/ffi_struct'
+require 'rbconfig'
 
-# Minimal support needed to run ffi/library
 module FFI
-  Platform = Truffle::FFI::Platform
+  AbstractMemory = ::Truffle::FFI::AbstractMemory
+  Pointer = ::Truffle::FFI::Pointer
+  MemoryPointer = ::Truffle::FFI::MemoryPointer
+  NullPointerError = ::Truffle::FFI::NullPointerError
 
-  class DynamicLibrary
-    RTLD_LAZY   = Truffle::Config['platform.dlopen.RTLD_LAZY']
-    RTLD_NOW    = Truffle::Config['platform.dlopen.RTLD_NOW']
-    RTLD_GLOBAL = Truffle::Config['platform.dlopen.RTLD_GLOBAL']
-    RTLD_LOCAL  = Truffle::Config['platform.dlopen.RTLD_LOCAL']
-
-    attr_reader :name
-
-    def self.open(libname, flags)
-      code = libname ? "load '#{libname}'" : 'default'
-      handle = Truffle::Interop.eval('application/x-native', code)
-      DynamicLibrary.new(libname, handle)
-    end
-
-    def initialize(name, handle)
-      @name = name
-      @handle = handle
-    end
-
-    def find_symbol(name)
-      @handle[name]
-    end
-
-    def inspect
-      "\#<#{self.class} @name=#{@name.inspect}>"
+  # Redefine Pointer.find_type_size to consider FFI types
+  class Pointer
+    def self.find_type_size(type)
+      ::FFI.type_size(::FFI.find_type(type))
     end
   end
 
-  def self.find_type(*args)
-    Truffle::FFI.find_type(*args)
+  module Platform
+    CPU = RbConfig::CONFIG['host_cpu']
+    # eregon: I would like to use rbconfig/sizeof here, but the linker requires
+    # ffi, and the linker is needed to build the rbconfig/sizeof C extension,
+    # so we need to break the cycle.
+    ADDRESS_SIZE = 64
+    LONG_SIZE = 64
   end
 
-  def self.type_size(*args)
-    Truffle::FFI.type_size(*args)
-  end
+  TypeDefs = {}
 end
 
-require_relative 'ffi/library'
-
-module FFI
-  module Library
-    LIBC = Truffle::LIBC
-
-    # Indicies are based on the NativeTypes enum
-    TO_NFI_TYPE = [
-      'SINT8',  # char
-      'UINT8',  # uchar
-      'UINT8',  # bool
-      'SINT16', # short
-      'UINT16', # ushort
-      'SINT32', # int
-      'UINT32', # uint
-      'SINT64', # long
-      'UINT64', # ulong
-      'SINT64', # ll
-      'UINT64', # ull
-      'FLOAT',  # float
-      'DOUBLE', # double
-      'POINTER',# ptr
-      'VOID',   # void
-      'STRING', # string
-      # strptr
-      # chararr
-      # enum
-      # varargs
-    ]
-
-    private def to_nfi_type(type)
-      idx = FFI.find_type(type)
-      TO_NFI_TYPE.fetch(idx)
-    end
-
-    def attach_function(method_name, native_name, args_types, return_type = nil, options = {})
-      unless return_type && (String === native_name || Symbol === native_name)
-        native_name, args_types, return_type, options = method_name, native_name, args_types, return_type
-        options ||= {}
-      end
-
-      warn "options #{options.inspect} ignored for attach_function :#{method_name}" unless options.empty?
-
-      nfi_args_types = args_types.map { |type| to_nfi_type(type) }
-      nfi_return_type = to_nfi_type(return_type)
-
-      function = @ffi_libs.each do |library|
-        break library.find_symbol(native_name)
-      end
-      signature = "(#{nfi_args_types.join(',')}):#{nfi_return_type}"
-      function = function.bind(Truffle::Interop.to_java_string(signature))
-
-      define_singleton_method(method_name) do |*args|
-        result = function.call(*args)
-        if return_type == :pointer
-          FFI::Pointer.new(Truffle::Interop.unbox(result))
-        else
-          result
-        end
-      end
-    end
-
-    def find_type(t)
-      FFI.find_type(t)
-    end
-  end
-
-  Pointer = Truffle::FFI::Pointer
-  MemoryPointer = Truffle::FFI::MemoryPointer
-  Struct = Truffle::FFI::Struct
-  Union = Truffle::FFI::Union
-
-  class Struct
-    def self.ptr
-      warn "validation for #{self} parameter not yet implemented"
-      :pointer
-    end
-  end
-end
+# Require the pure-Ruby Truffle NFI backend
+require_relative 'truffle/ffi_backend/last_error'
+require_relative 'truffle/ffi_backend/data_converter'
+require_relative 'truffle/ffi_backend/type'
+require_relative 'truffle/ffi_backend/struct_layout'
+require_relative 'truffle/ffi_backend/struct'
+require_relative 'truffle/ffi_backend/buffer'
+require_relative 'truffle/ffi_backend/function'
+require_relative 'truffle/ffi_backend/dynamic_library'
+
+# Require the FFI gem Ruby files
+require_relative 'ffi/ffi'