Merge branch 'master' into trim-experimental

Author: arnavk23

contrib/juliac-buildscript.jl

@@ -1,9 +1,5 @@
 # Script to run in the process that generates juliac's object file output
 
-inputfile = ARGS[1]
-output_type = ARGS[2]
-add_ccallables = ARGS[3] == "true"
-
 # Run the verifier in the current world (before modifications), so that error
 # messages and types print in their usual way.
 Core.Compiler._verify_trim_world_age[] = Base.get_world_counter()
@@ -189,13 +185,37 @@ end
 
 import Base.Experimental.entrypoint
 
-let mod = Base.include(Base.__toplevel__, inputfile)
-    if !isa(mod, Module)
-        mod = Main
-    end
+# for use as C main if needed
+function _main(argc::Cint, argv::Ptr{Ptr{Cchar}})::Cint
+    args = ccall(:jl_set_ARGS, Any, (Cint, Ptr{Ptr{Cchar}}), argc, argv)::Vector{String}
+    return Main.main(args)
+end
+
+let mod = Base.include(Main, ARGS[1])
     Core.@latestworld
-    if output_type == "--output-exe" && isdefined(mod, :main) && !add_ccallables
-        entrypoint(mod.main, ())
+    if ARGS[2] == "--output-exe"
+        have_cmain = false
+        if isdefined(Main, :main)
+            for m in methods(Main.main)
+                if isdefined(m, :ccallable)
+                    # TODO: possibly check signature and return type
+                    have_cmain = true
+                    break
+                end
+            end
+        end
+        if !have_cmain
+            if Base.should_use_main_entrypoint()
+                if hasmethod(Main.main, Tuple{Vector{String}})
+                    entrypoint(_main, (Cint, Ptr{Ptr{Cchar}}))
+                    Base._ccallable("main", Cint, Tuple{typeof(_main), Cint, Ptr{Ptr{Cchar}}})
+                else
+                    error("`@main` must accept a `Vector{String}` argument.")
+                end
+            else
+                error("To generate an executable a `@main` function must be defined.")
+            end
+        end
     end
     #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{Base.SubString{String}, 1}, String))
     #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{String, 1}, Char))
@@ -204,7 +224,7 @@ let mod = Base.include(Base.__toplevel__, inputfile)
     entrypoint(Base.wait_forever, ())
     entrypoint(Base.trypoptask, (Base.StickyWorkqueue,))
     entrypoint(Base.checktaskempty, ())
-    if add_ccallables
+    if ARGS[3] == "true"
         ccall(:jl_add_ccallable_entrypoints, Cvoid, ())
     end
 end

contrib/print_sorted_stdlibs.jl

@@ -12,12 +12,13 @@ function check_flag(flag)
 end
 
 if check_flag("--help") || check_flag("-h")
-    println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls] [--exclude-sysimage]")
+    println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls] [--exclude-sysimage] [--only-sysimg]")
 end
 
 # Allow users to ask for JLL or no JLLs
 exclude_jlls = check_flag("--exclude-jlls")
 exclude_sysimage = check_flag("--exclude-sysimage")
+only_sysimage = check_flag("--only-sysimg")
 
 # Default to the `stdlib/vX.Y` directory
 STDLIB_DIR = get(ARGS, 1, joinpath(@__DIR__, "..", "usr", "share", "julia", "stdlib"))
@@ -81,9 +82,19 @@ if exclude_jlls
     filter!(p -> !endswith(p, "_jll"), sorted_projects)
 end
 
-if exclude_sysimage
-    loaded_modules = Set(map(k->k.name, collect(keys(Base.loaded_modules))))
-    filter!(p->!in(p, loaded_modules), sorted_projects)
+if only_sysimage && exclude_sysimage
+    println(stderr, "Warning: --only-sysimg and --exclude-sysimage are mutually exclusive. Prioritizing --only-sysimg.")
+    exclude_sysimage = false
+end
+
+if only_sysimage || exclude_sysimage
+    loaded_modules_set = Set(map(k->k.name, collect(keys(Base.loaded_modules))))
+
+    if only_sysimage
+        filter!(p -> in(p, loaded_modules_set), sorted_projects)
+    else
+        filter!(p -> !in(p, loaded_modules_set), sorted_projects)
+    end
 end
 
 # Print out sorted projects, ready to be pasted into `sysimg.jl`
@@ -92,6 +103,9 @@ println("    # Stdlibs sorted in dependency, then alphabetical, order by contrib
 if exclude_jlls
     println("    # Run with the `--exclude-jlls` option to filter out all JLL packages")
 end
+if only_sysimage
+    println("    # Run with the `--only-sysimg` option to filter for only packages included in the system image")
+end
 if exclude_sysimage
     println("    # Run with the `--exclude-sysimage` option to filter out all packages included in the system image")
 end

doc/Makefile

@@ -34,6 +34,9 @@ deps: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt
 	$(JLCHECKSUM) "$<"
 	cp "$<" UnicodeData.txt
 
+alldeps: deps
+	$(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) deps
+
 checksum-unicodedata: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt
 	$(JLCHECKSUM) "$<"
 

doc/README.md

@@ -27,3 +27,11 @@ $ make -C doc doctest=true
 ```
 
 from the root directory.
+
+## Customizing Doctest Execution
+
+By default, doctests are run using the in-tree Julia executable.
+This behavior can be changed by setting the `JULIA_EXECUTABLE` Makefile variable.
+
+> [!WARNING]
+> Using a custom `JULIA_EXECUTABLE` will not pick up changes to docstrings for Base or any standard library built into the system image. To see the list of standard libraries that are part of the system image, you can run the `contrib/print_sorted_stdlibs.jl` script (e.g., `julia contrib/print_sorted_stdlibs.jl --only-sysimg`).

doc/make.jl

@@ -8,6 +8,10 @@ push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia"))
 using Pkg
 Pkg.instantiate()
 
+if "deps" in ARGS
+    exit()
+end
+
 using Documenter
 import LibGit2
 

src/gf.c

@@ -4642,8 +4642,6 @@ JL_DLLEXPORT void jl_extern_c(jl_value_t *name, jl_value_t *declrt, jl_tupletype
         jl_error("@ccallable: function object must be a singleton");
 
     // compute / validate return type
-    if (!jl_is_concrete_type(declrt) || jl_is_kind(declrt))
-        jl_error("@ccallable: return type must be concrete and correspond to a C type");
     if (!jl_type_mappable_to_c(declrt))
         jl_error("@ccallable: return type doesn't correspond to a C type");
 

src/jlapi.c

@@ -50,24 +50,28 @@ JL_DLLEXPORT int jl_is_initialized(void)
  * @param argc The number of command line arguments.
  * @param argv Array of command line arguments.
  */
-JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv)
+JL_DLLEXPORT jl_value_t *jl_set_ARGS(int argc, char **argv)
 {
-    if (jl_core_module != NULL) {
-        jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS"));
-        if (args == NULL) {
-            args = jl_alloc_vec_any(0);
-            JL_GC_PUSH1(&args);
+    jl_array_t *args = NULL;
+    jl_value_t *vecstr = NULL;
+    JL_GC_PUSH2(&args, &vecstr);
+    if (jl_core_module != NULL)
+        args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS"));
+    if (args == NULL) {
+        vecstr = jl_apply_array_type((jl_value_t*)jl_string_type, 1);
+        args = jl_alloc_array_1d(vecstr, 0);
+        if (jl_core_module != NULL)
             jl_set_const(jl_core_module, jl_symbol("ARGS"), (jl_value_t*)args);
-            JL_GC_POP();
-        }
-        assert(jl_array_nrows(args) == 0);
-        jl_array_grow_end(args, argc);
-        int i;
-        for (i = 0; i < argc; i++) {
-            jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]);
-            jl_array_ptr_set(args, i, s);
-        }
     }
+    assert(jl_array_nrows(args) == 0);
+    jl_array_grow_end(args, argc);
+    int i;
+    for (i = 0; i < argc; i++) {
+        jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]);
+        jl_array_ptr_set(args, i, s);
+    }
+    JL_GC_POP();
+    return (jl_value_t*)args;
 }
 
 JL_DLLEXPORT void jl_init_with_image_handle(void *handle) {

src/julia.h

@@ -2579,7 +2579,7 @@ uint64_t parse_heap_size_hint(const char *optarg, const char *option_name);
 
 // Set julia-level ARGS array according to the arguments provided in
 // argc/argv
-JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv);
+JL_DLLEXPORT jl_value_t *jl_set_ARGS(int argc, char **argv);
 
 JL_DLLEXPORT int jl_generating_output(void) JL_NOTSAFEPOINT;
 

src/staticdata.c

@@ -2638,12 +2638,12 @@ static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED
     jl_gc_wb(m, jl_atomic_load_relaxed(&bindingkeyset2));
 }
 
-static void strip_slotnames(jl_array_t *slotnames)
+static void strip_slotnames(jl_array_t *slotnames, int n)
 {
     // replace slot names with `?`, except unused_sym since the compiler looks at it
     jl_sym_t *questionsym = jl_symbol("?");
-    int i, l = jl_array_len(slotnames);
-    for (i = 0; i < l; i++) {
+    int i;
+    for (i = 0; i < n; i++) {
         jl_value_t *s = jl_array_ptr_ref(slotnames, i);
         if (s != (jl_value_t*)jl_unused_sym)
             jl_array_ptr_set(slotnames, i, questionsym);
@@ -2662,7 +2662,7 @@ static jl_value_t *strip_codeinfo_meta(jl_method_t *m, jl_value_t *ci_, jl_code_
     else {
         ci = (jl_code_info_t*)ci_;
     }
-    strip_slotnames(ci->slotnames);
+    strip_slotnames(ci->slotnames, jl_array_len(ci->slotnames));
     ci->debuginfo = jl_nulldebuginfo;
     jl_gc_wb(ci, ci->debuginfo);
     jl_value_t *ret = (jl_value_t*)ci;
@@ -2736,7 +2736,11 @@ static int strip_all_codeinfos__(jl_typemap_entry_t *def, void *_env)
             }
             jl_array_t *slotnames = jl_uncompress_argnames(m->slot_syms);
             JL_GC_PUSH1(&slotnames);
-            strip_slotnames(slotnames);
+            int tostrip = jl_array_len(slotnames);
+            // for keyword methods, strip only nargs to keep the keyword names at the end for reflection
+            if (jl_tparam0(jl_unwrap_unionall(m->sig)) == jl_typeof(jl_kwcall_func))
+                tostrip = m->nargs;
+            strip_slotnames(slotnames, tostrip);
             m->slot_syms = jl_compress_argnames(slotnames);
             jl_gc_wb(m, m->slot_syms);
             JL_GC_POP();

test/cmdlineargs.jl

@@ -1258,3 +1258,13 @@ end
     timeout = 120
     @test parse(Int,read(`$exename --timeout-for-safepoint-straggler=$timeout -E "Base.JLOptions().timeout_for_safepoint_straggler_s"`, String)) == timeout
 end
+
+@testset "--strip-metadata" begin
+    mktempdir() do dir
+        @test success(pipeline(`$(Base.julia_cmd()) --strip-metadata -t1,0 --output-o $(dir)/sys.o.a -e 0`, stderr=stderr, stdout=stdout))
+        if isfile(joinpath(dir, "sys.o.a"))
+            Base.Linking.link_image(joinpath(dir, "sys.o.a"), joinpath(dir, "sys.so"))
+            @test readchomp(`$(Base.julia_cmd()) -t1,0 -J $(dir)/sys.so -E 'hasmethod(sort, (Vector{Int},), (:dims,))'`) == "true"
+        end
+    end
+end

test/trimming/basic_jll.jl

@@ -1,14 +1,10 @@
-module MyApp
-
 using Libdl
 using Zstd_jll
 
-Base.@ccallable function main()::Cint
+function @main(args::Vector{String})::Cint
     println(Core.stdout, "Julia! Hello, world!")
     fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString)
     println(Core.stdout, unsafe_string(ccall(fptr, Cstring, ())))
     println(Core.stdout, unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ())))
     return 0
 end
-
-end

test/trimming/hello.jl

@@ -1,13 +1,10 @@
-module MyApp
-
 world::String = "world!"
 const str = OncePerProcess{String}() do
     return "Hello, " * world
 end
 
-Base.@ccallable function main()::Cint
+function @main(args::Vector{String})::Cint
     println(Core.stdout, str())
+    foreach(x->println(Core.stdout, x), args)
     return 0
 end
-
-end

test/trimming/trimming.jl

@@ -6,7 +6,7 @@ bindir = dirname(ARGS[1])
 let exe_suffix = splitext(Base.julia_exename())[2]
 
     hello_exe = joinpath(bindir, "hello" * exe_suffix)
-    @test readchomp(`$hello_exe`) == "Hello, world!"
+    @test readchomp(`$hello_exe arg1 arg2`) == "Hello, world!\n$hello_exe\narg1\narg2"
     @test filesize(hello_exe) < 2_000_000
 
     basic_jll_exe = joinpath(bindir, "basic_jll" * exe_suffix)