From 45c1476444495b2a42bdcc265bab7d4509ad15ac Mon Sep 17 00:00:00 2001 From: fchrstou Date: Wed, 21 Jun 2023 19:39:22 +0200 Subject: [PATCH 1/4] Support shared sessions --- .github/workflows/CI.yml | 1 - .gitignore | 1 + Project.toml | 1 + docs/Manifest.toml | 151 +++++++++++++++++++++++++++++++++++---- docs/Project.toml | 1 + docs/src/howto.md | 59 +++++++++++++++ docs/src/reference.md | 2 + src/RemoteREPL.jl | 7 +- src/client.jl | 84 +++++++++++++++++----- src/server.jl | 52 ++++++++------ src/tunnels.jl | 2 - test/runtests.jl | 3 +- 12 files changed, 308 insertions(+), 56 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6c943c9..0c3ee10 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -83,7 +83,6 @@ jobs: version: '1.6' - run: julia --project=docs -e ' using Pkg; - Pkg.develop(PackageSpec(; path=pwd())); Pkg.instantiate();' - run: julia --project=docs docs/make.jl env: diff --git a/.gitignore b/.gitignore index b067edd..db7c9f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /Manifest.toml +docs/build/ diff --git a/Project.toml b/Project.toml index ace7acb..0ba1499 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,7 @@ REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" ReplMaker = "b873ce64-0db9-51f5-a568-4457d8e49576" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] OpenSSH_jll = "8.1, 9" diff --git a/docs/Manifest.toml b/docs/Manifest.toml index a3f3dc8..8572030 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,5 +1,17 @@ # This file is machine-generated - editing it directly is not advised +[[ANSIColoredPrinters]] +git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" +uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9" +version = "0.0.1" + +[[ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -9,36 +21,68 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[DocStringExtensions]] deps = ["LibGit2"] -git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f" +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.8.5" +version = "0.9.3" [[Documenter]] -deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] -git-tree-sha1 = "621850838b3e74dd6dd047b5432d2e976877104e" +deps = ["ANSIColoredPrinters", "Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] +git-tree-sha1 = "58fea7c536acd71f3eef6be3b21c0df5f3df88fd" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.27.2" +version = "0.27.24" + +[[Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" [[IOCapture]] deps = ["Logging", "Random"] -git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" +git-tree-sha1 = "d75853a0bdbfb1ac815478bacd89cd27b550ace6" uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" -version = "0.2.2" +version = "0.2.3" [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +[[JLLWrappers]] +deps = ["Preferences"] +git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.4.1" + [[JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4" +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.1" +version = "0.21.4" + +[[LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" [[LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +[[LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + [[Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -46,17 +90,56 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +[[MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" + [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +[[MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" + [[NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[OpenSSH_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "1b2f042897343a9dfdcc9366e4ecbd3d00780c49" +uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" +version = "8.9.0+1" + +[[OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1aa4b74f80b01c6bc2b89992b861b5f210e665b5" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "1.1.21+0" [[Parsers]] -deps = ["Dates"] -git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc" +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "4b2e829ee66d4218e0cef22c0a64ee37cf258c29" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "1.1.0" +version = "2.7.1" + +[[Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.0" + +[[PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.1.2" + +[[Preferences]] +deps = ["TOML"] +git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.0" [[Printf]] deps = ["Unicode"] @@ -67,11 +150,24 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[Random]] -deps = ["Serialization"] +deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +[[RemoteREPL]] +deps = ["Logging", "OpenSSH_jll", "REPL", "ReplMaker", "Serialization", "Sockets", "UUIDs"] +path = ".." +uuid = "1bd9f7bb-701c-4338-bec7-ac987af7c555" +version = "0.2.17" + +[[ReplMaker]] +deps = ["REPL", "Unicode"] +git-tree-sha1 = "f8bb680b97ee232c4c6591e213adc9c1e4ba0349" +uuid = "b873ce64-0db9-51f5-a568-4457d8e49576" +version = "0.2.7" + [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -79,9 +175,38 @@ uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +[[TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + [[Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + [[Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" + +[[nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/docs/Project.toml b/docs/Project.toml index dfa65cd..c403dba 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,2 +1,3 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +RemoteREPL = "1bd9f7bb-701c-4338-bec7-ac987af7c555" diff --git a/docs/src/howto.md b/docs/src/howto.md index 051962b..fbece02 100644 --- a/docs/src/howto.md +++ b/docs/src/howto.md @@ -107,6 +107,17 @@ julia@localhost> a_variable 1 ``` + +## Use common session among different clients +A session, as implemented in `ServerSideSession`, describes the display properties and the module under which commands are evaluated. +Multiple clients can share same such properties by using the same `session_id`. For example: + +```julia +julia> session_id = UUID("f03aec15-3e14-4d58-bcfa-82f8d33c9f9a") + +julia> connect_repl(; session_id=session_id) +``` + ## Use alternatives to SSH ### AWS Session Manager @@ -128,6 +139,53 @@ connect_remote(); ``` which will allow you to use `@remote` without the REPL mode. +### More on Pluto +Pluto presents a peculiarity as the default module is constantly changing. +In order to closely track the newest notebook state, you will need to tap into the client's session and update the module. +You could write the following code in the pluto notebook that updates the module every second (if you have a better event-driven update solution, please raise an issue!). + +```julia +using PlutoLinks, PlutoHooks + +using RemoteREPL, Sockets, UUIDs + +server = Sockets.listen(Sockets.localhost, 27765) + +@async serve_repl(server) + +session_id = UUID("f03aec15-3e14-4d58-bcfa-82f8d33c9f9a") + +con2server = connect_remote(Sockets.localhost, 27765; session_id=session_id) + +takemodulesymbol() = Symbol("workspace#" ,PlutoRunner.moduleworkspace_count[]) + +let # update module in RemoteREPL every 1 sec + count, set_count = @use_state(1) + @use_task([]) do + new_count = count + while true + sleep(1.0) + new_count += 1 + set_count(new_count) # (this will trigger a re-run) + end + end + mod = eval(takemodulesymbol()) + #@eval(@remoterepl $"%module $mod") + remote_module!(mod) +end +``` + +Then open a repl and do: + +```julia +julia> using RemoteREPL, Sockets, UUIDs + +julia> connect_repl(Sockets.localhost, 27765; session_id=UUID("f03aec15-3e14-4d58-bcfa-82f8d33c9f9a")) +``` + +Since the session's module is being regularly updated by the Pluto notebook, your REPL will be in sync with the notebook's state. + + ## Troubleshooting connection issues Sometimes errors will be encountered. This section aims to show some errors experienced by users, and what the underlying problem was. We will use some terms in this section, introduced in the table below. |Term|Explanation| @@ -153,3 +211,4 @@ any group or other users. On a linux system, this is accomplished by running the chmod go-w /home/username/.ssh/* ``` If you are using a different operating system, please google how to remove write permissions on files, and try to do the same thing. + diff --git a/docs/src/reference.md b/docs/src/reference.md index 4061fca..11c2919 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -42,5 +42,7 @@ serve_repl connect_remote RemoteREPL.@remote RemoteREPL.remote_eval +RemoteREPL.run_remote_repl_command +RemoteREPL.remote_module! ``` diff --git a/src/RemoteREPL.jl b/src/RemoteREPL.jl index b22427c..f010c7c 100644 --- a/src/RemoteREPL.jl +++ b/src/RemoteREPL.jl @@ -1,6 +1,11 @@ module RemoteREPL -export connect_repl, serve_repl, @remote, connect_remote +using REPL, ReplMaker +using Sockets, Serialization +using UUIDs, Logging +using OpenSSH_jll + +export connect_repl, serve_repl, @remote, connect_remote, run_remote_repl_command, remote_module! const DEFAULT_PORT = 27754 const PROTOCOL_MAGIC = "RemoteREPL" diff --git a/src/client.jl b/src/client.jl index 90ab30d..a616bfd 100644 --- a/src/client.jl +++ b/src/client.jl @@ -1,8 +1,3 @@ -using ReplMaker -using REPL -using Serialization -using Sockets - struct RemoteException <: Exception msg::String end @@ -114,7 +109,8 @@ mutable struct Connection region::Union{AbstractString,Nothing} namespace::Union{AbstractString,Nothing} socket::Union{IO,Nothing} - in_module::Symbol + in_module::Union{Symbol, Expr} + session_id::UUID end function Connection(; host::Union{AbstractString,Sockets.IPAddr}=Sockets.localhost, @@ -123,8 +119,11 @@ function Connection(; host::Union{AbstractString,Sockets.IPAddr}=Sockets.localho ssh_opts::Cmd=``, region=nothing, namespace=nothing, - in_module::Symbol=:Main) - conn = Connection(host, port, tunnel, ssh_opts, region, namespace, nothing, in_module) + in_module::Symbol=:Main, + session_id=nothing) + sesid = isnothing(session_id) ? UUIDs.uuid4() : session_id + @info "Using session id $(sesid)" + conn = Connection(host, port, tunnel, ssh_opts, region, namespace, nothing, in_module, sesid) setup_connection!(conn) finalizer(close, conn) end @@ -140,6 +139,8 @@ function setup_connection!(conn::Connection) namespace=conn.namespace) end Sockets.nagle(socket, false) # Disables nagles algorithm. Appropriate for interactive connections. + # transmit session id + serialize(socket, conn.session_id) try verify_header(socket) catch exc @@ -306,7 +307,13 @@ function REPL.complete_line(provider::RemoteCompletionProvider, return result[2] end -function run_remote_repl_command(conn, out_stream, cmdstr) +""" + run_remote_repl_command(conn::Connection, out_stream::IO, cmdstr::String) + +Evaluate `cmdstr` in the remote session of connection `conn` and write result into `out_stream`. +Also supports the magic `RemoteREPL` commands like `%module` and `%include`. +""" +function run_remote_repl_command(conn::Connection, out_stream::IO, cmdstr::String) # Compute command magic = match_magic_syntax(cmdstr) if isnothing(magic) @@ -375,6 +382,45 @@ function run_remote_repl_command(conn, out_stream, cmdstr) return result_for_display end +""" + run_remote_repl_command(cmdstr::String) + +Evaluate `cmdstr` in the last opened RemoteREPL connection and print result to `Base.stdout` +""" +run_remote_repl_command(cmd::String) = run_remote_repl_command(_repl_client_connection, Base.stdout, cmd) + +""" + run_remote_repl_command(conn::Connection, cmdstr::String) + +Evaluate `cmdstr` in the connection `conn` and print result to `Base.stdout`. +""" +run_remote_repl_command(conn::Connection, cmd::String) = run_remote_repl_command(conn, Base.stdout, cmd) + +""" + remote_module!(mod::Module, conn::Connection = _repl_client_connection) + +Change future remote commands in the session of connection `conn` to be evaluated into module `mod`. +The default connection `_repl_client_connection` is the last established RemoteREPL connection. +If the module cannot be evaluated locally pass the name as a string. +Equivalent to using the `%module` magic. +""" +function remote_module!(mod::Module, conn=_repl_client_connection) + run_remote_repl_command(conn, Base.stdout, "%module $(mod)") +end + +""" + remote_module!(modstr::String, conn::Connection = _repl_client_connection) + +Change future remote commands in the session of connection `conn` to be evaluated into module identified by `modstr`. +The default connection `_repl_client_connection` is the last established RemoteREPL connection. +Equivalent to using the `%module` magic. +""" +function remote_module!(modstr::String, conn=_repl_client_connection) + run_remote_repl_command(conn, Base.stdout, "%module "*modstr) +end + + + remote_eval_and_fetch(::Nothing, ex) = error("No remote connection is active") function remote_eval_and_fetch(conn::Connection, ex) @@ -412,7 +458,7 @@ _repl_client_connection = nothing """ connect_repl([host=localhost,] port::Integer=$DEFAULT_PORT; use_ssh_tunnel = (host != localhost) ? :ssh : :none, - ssh_opts = ``, repl=Base.active_repl) + ssh_opts = ``, repl=Base.active_repl, session_id = nothing) Connect client REPL to a remote `host` on `port`. This is then accessible as a remote sub-repl of the current Julia session. @@ -441,9 +487,10 @@ function connect_repl(host=Sockets.localhost, port::Integer=DEFAULT_PORT; region::Union{AbstractString,Nothing}=nothing, namespace::Union{AbstractString,Nothing}=nothing, startup_text::Bool=true, - repl=Base.active_repl) + repl=Base.active_repl, + session_id=nothing) - conn = connect_remote(host, port; tunnel, ssh_opts, region,namespace) + conn = connect_remote(host, port; tunnel, ssh_opts, region, namespace, session_id) out_stream = stdout prompt = ReplMaker.initrepl(c->run_remote_repl_command(conn, out_stream, c), repl = Base.active_repl, @@ -464,7 +511,7 @@ connect_repl(port::Integer) = connect_repl(Sockets.localhost, port) """ connect_remote([host=localhost,] port::Integer=$DEFAULT_PORT; tunnel = (host != localhost) ? :ssh : :none, - ssh_opts = ``) + ssh_opts = ``, session_id = nothing) Connect to remote server without any REPL integrations. This will allow you to use `@remote`, but not the REPL mode. Useful in circumstances where no REPL is available, but interactivity is desired like Jupyter or Pluto notebooks. @@ -474,7 +521,8 @@ function connect_remote(host=Sockets.localhost, port::Integer=DEFAULT_PORT; tunnel::Symbol = host!=Sockets.localhost ? :ssh : :none, ssh_opts::Cmd=``, region::Union{AbstractString,Nothing}=nothing, - namespace::Union{AbstractString,Nothing}=nothing) + namespace::Union{AbstractString,Nothing}=nothing, + session_id=nothing) global _repl_client_connection @@ -486,7 +534,7 @@ function connect_remote(host=Sockets.localhost, port::Integer=DEFAULT_PORT; end end conn = RemoteREPL.Connection(host=host, port=port, tunnel=tunnel, - ssh_opts=ssh_opts, region=region, namespace=namespace) + ssh_opts=ssh_opts, region=region, namespace=namespace, session_id = session_id) # Record the connection in a global variable so it's accessible to REPL and `@remote` _repl_client_connection = conn @@ -542,7 +590,7 @@ _remote_expr(conn, ex) = :(remote_eval_and_fetch($conn, $(QuoteNode(ex)))) remote_eval(cmdstr) remote_eval(host, port, cmdstr) -Parse a string `cmdstr`, evaluate it in the remote REPL server's `Main` module, +Parse a string `cmdstr`, evaluate it in the remote REPL server's `Main` module or the session with `session_id`, then close the connection. Returns the result which the REPL would normally pass to `show()` (likely a `Text` object). @@ -554,8 +602,8 @@ RemoteREPL.remote_eval("exit()") ``` """ function remote_eval(host, port::Integer, cmdstr::AbstractString; - tunnel::Symbol = host!=Sockets.localhost ? :ssh : :none) - conn = Connection(; host=host, port=port, tunnel=tunnel) + tunnel::Symbol = host!=Sockets.localhost ? :ssh : :none, session_id=nothing) + conn = Connection(; host=host, port=port, tunnel=tunnel, session_id=session_id) local result try setup_connection!(conn) diff --git a/src/server.jl b/src/server.jl index f05c4c4..bef1ac9 100644 --- a/src/server.jl +++ b/src/server.jl @@ -1,16 +1,15 @@ -using Sockets -using Serialization -using REPL -using Logging - mutable struct ServerSideSession - socket + sockets::Vector display_properties::Dict in_module::Module end -Base.isopen(session::ServerSideSession) = isopen(session.socket) -Base.close(session::ServerSideSession) = close(session.socket) +Base.isopen(session::ServerSideSession) = any(isopen.(session.sockets)) + +function close_and_delete!(session::ServerSideSession, socket) + close(socket) + filter!(!=(socket), session.sockets) +end function send_header(io, ser_version=Serialization.ser_version) write(io, PROTOCOL_MAGIC, PROTOCOL_VERSION) @@ -177,8 +176,7 @@ function serialize_responses(socket, response_chan) end # Serve a remote REPL session to a single client -function serve_repl_session(session) - socket = session.socket +function serve_repl_session(session, socket) send_header(socket) @sync begin request_chan = Channel(1) @@ -248,29 +246,43 @@ end serve_repl(port::Integer; kws...) = serve_repl(Sockets.localhost, port; kws...) function serve_repl(server::Base.IOServer; on_client_connect=nothing) - open_sessions = Set{ServerSideSession}() + open_sessions = Dict{UUID, ServerSideSession}() + session_lock = Base.ReentrantLock() @sync try while isopen(server) socket = accept(server) - session = ServerSideSession(socket, Dict(), Main) - push!(open_sessions, session) + + session, session_id, socketidx = lock(session_lock) do + # expect session id + session_id = deserialize(socket) + session = if haskey(open_sessions, session_id) + push!(open_sessions[session_id].sockets, socket) + open_sessions[session_id] + else + open_sessions[session_id] = ServerSideSession([socket], Dict(), Main) + end + session, session_id, length(session.sockets) + end + peer = getpeername(socket) @async try if !isnothing(on_client_connect) on_client_connect(session) end - serve_repl_session(session) + serve_repl_session(session, socket) catch exc - if !(exc isa EOFError && !isopen(session)) + if !(exc isa EOFError && !isopen(socket)) @warn "Something went wrong evaluating client command" #= =# exception=exc,catch_backtrace() end finally @info "REPL client exited" peer - close(session) - pop!(open_sessions, session) + close_and_delete!(session, socket) + lock(session_lock) do + length(session.sockets) == 0 && delete!(open_sessions, session_id) + end end - @info "REPL client opened a connection" peer + @info "REPL client opened a connection with session id $(session_id)" peer end catch exc if exc isa Base.IOError && !isopen(server) @@ -280,8 +292,8 @@ function serve_repl(server::Base.IOServer; on_client_connect=nothing) @error "Unexpected server failure" isopen(server) exception=exc,catch_backtrace() rethrow() finally - for session in open_sessions - close(session) + for session in values(open_sessions) + foreach(close, session.sockets) end end end diff --git a/src/tunnels.jl b/src/tunnels.jl index c9b3d85..263d8b1 100644 --- a/src/tunnels.jl +++ b/src/tunnels.jl @@ -1,7 +1,5 @@ # Utilities for securely tunnelling traffic from client to a remote server -using OpenSSH_jll - # Find a free port on `network_interface` function find_free_port(network_interface) # listen on port 0 => kernel chooses a free port. See, for example, diff --git a/test/runtests.jl b/test/runtests.jl index e675e8e..6e1579e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,7 @@ using RemoteREPL using Test using Sockets using RemoteREPL: repl_prompt_text, match_magic_syntax, DEFAULT_PORT +using UUIDs ENV["JULIA_DEBUG"] = "RemoteREPL" @@ -52,7 +53,7 @@ end function fake_conn(host, port; is_open=true) io = IOBuffer() is_open || close(io) - RemoteREPL.Connection(host, port, :none, ``, nothing, nothing, io, :Main) + RemoteREPL.Connection(host, port, :none, ``, nothing, nothing, io, :Main, uuid4()) end @test repl_prompt_text(fake_conn(Sockets.localhost, DEFAULT_PORT)) == "julia@localhost> " @test repl_prompt_text(fake_conn("localhost", DEFAULT_PORT)) == "julia@localhost> " From 14ddf46fe2920b60bd68517b60cf9838873096d3 Mon Sep 17 00:00:00 2001 From: fchrstou Date: Mon, 3 Jul 2023 13:52:04 +0200 Subject: [PATCH 2/4] Added tests and review comments --- .github/workflows/CI.yml | 1 + bin/julia-r | 2 +- docs/src/howto.md | 9 +++++++++ docs/src/reference.md | 2 +- src/RemoteREPL.jl | 2 +- src/client.jl | 30 ++++++++++++++---------------- src/server.jl | 9 ++++----- test/runtests.jl | 24 ++++++++++++++++++------ 8 files changed, 49 insertions(+), 30 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0c3ee10..2f6d59d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -73,6 +73,7 @@ jobs: - uses: julia-actions/julia-runtest@v1 env: SSH_AUTH_SOCK: /tmp/ssh_agent.sock + - run: julia -e "import Pkg; Pkg.add(\"TestEnv\")" docs: name: Documentation runs-on: ubuntu-latest diff --git a/bin/julia-r b/bin/julia-r index be8881e..f14db85 100755 --- a/bin/julia-r +++ b/bin/julia-r @@ -19,7 +19,7 @@ atreplinit() do repl # Run one command as part of connection setup to trigger compilation # This makes the REPL more immediately responsive after it prints the # welcome message. - RemoteREPL.run_remote_repl_command(RemoteREPL._repl_client_connection, + remotecmd(RemoteREPL._repl_client_connection, stdout, "\"hi\"") println(""" Connected to $host diff --git a/docs/src/howto.md b/docs/src/howto.md index fbece02..5c3b024 100644 --- a/docs/src/howto.md +++ b/docs/src/howto.md @@ -118,6 +118,15 @@ julia> session_id = UUID("f03aec15-3e14-4d58-bcfa-82f8d33c9f9a") julia> connect_repl(; session_id=session_id) ``` +## Pass a command non-interactively +To programmatically pass a command to the remote julia kernel use [`remotecmd`](@ref). For example: + +```julia +julia> con2server = connect_remote(Sockets.localhost, 9093) # connect to port 9093 in localhost + +julia> remotecmd(con2server, "myvar = 1") # define a new var +``` + ## Use alternatives to SSH ### AWS Session Manager diff --git a/docs/src/reference.md b/docs/src/reference.md index 11c2919..7995035 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -42,7 +42,7 @@ serve_repl connect_remote RemoteREPL.@remote RemoteREPL.remote_eval -RemoteREPL.run_remote_repl_command +RemoteREPL.remotecmd RemoteREPL.remote_module! ``` diff --git a/src/RemoteREPL.jl b/src/RemoteREPL.jl index f010c7c..eb2f267 100644 --- a/src/RemoteREPL.jl +++ b/src/RemoteREPL.jl @@ -5,7 +5,7 @@ using Sockets, Serialization using UUIDs, Logging using OpenSSH_jll -export connect_repl, serve_repl, @remote, connect_remote, run_remote_repl_command, remote_module! +export connect_repl, serve_repl, @remote, connect_remote, remotecmd, remote_module! const DEFAULT_PORT = 27754 const PROTOCOL_MAGIC = "RemoteREPL" diff --git a/src/client.jl b/src/client.jl index a616bfd..c9ba61a 100644 --- a/src/client.jl +++ b/src/client.jl @@ -308,12 +308,12 @@ function REPL.complete_line(provider::RemoteCompletionProvider, end """ - run_remote_repl_command(conn::Connection, out_stream::IO, cmdstr::String) + remotecmd(conn::Connection, out_stream::IO, cmdstr::String) Evaluate `cmdstr` in the remote session of connection `conn` and write result into `out_stream`. Also supports the magic `RemoteREPL` commands like `%module` and `%include`. """ -function run_remote_repl_command(conn::Connection, out_stream::IO, cmdstr::String) +function remotecmd(conn::Connection, out_stream::IO, cmdstr::String) # Compute command magic = match_magic_syntax(cmdstr) if isnothing(magic) @@ -383,41 +383,39 @@ function run_remote_repl_command(conn::Connection, out_stream::IO, cmdstr::Strin end """ - run_remote_repl_command(cmdstr::String) + remotecmd(cmdstr::String) Evaluate `cmdstr` in the last opened RemoteREPL connection and print result to `Base.stdout` """ -run_remote_repl_command(cmd::String) = run_remote_repl_command(_repl_client_connection, Base.stdout, cmd) +remotecmd(cmd::String) = remotecmd(_repl_client_connection, Base.stdout, cmd) """ - run_remote_repl_command(conn::Connection, cmdstr::String) + remotecmd(conn::Connection, cmdstr::String) Evaluate `cmdstr` in the connection `conn` and print result to `Base.stdout`. """ -run_remote_repl_command(conn::Connection, cmd::String) = run_remote_repl_command(conn, Base.stdout, cmd) +remotecmd(conn::Connection, cmd::String) = remotecmd(conn, Base.stdout, cmd) """ - remote_module!(mod::Module, conn::Connection = _repl_client_connection) + remote_module!(conn::Connection = _repl_client_connection, mod::Module) Change future remote commands in the session of connection `conn` to be evaluated into module `mod`. The default connection `_repl_client_connection` is the last established RemoteREPL connection. If the module cannot be evaluated locally pass the name as a string. Equivalent to using the `%module` magic. """ -function remote_module!(mod::Module, conn=_repl_client_connection) - run_remote_repl_command(conn, Base.stdout, "%module $(mod)") -end +remote_module!(conn::Connection, mod::Module) = remotecmd(conn, Base.stdout, "%module $(mod)") +remote_module!(mod::Module) = remotecmd(_repl_client_connection, Base.stdout, "%module $(mod)") """ - remote_module!(modstr::String, conn::Connection = _repl_client_connection) + remote_module!(conn::Connection = _repl_client_connection, modstr::String) Change future remote commands in the session of connection `conn` to be evaluated into module identified by `modstr`. The default connection `_repl_client_connection` is the last established RemoteREPL connection. Equivalent to using the `%module` magic. """ -function remote_module!(modstr::String, conn=_repl_client_connection) - run_remote_repl_command(conn, Base.stdout, "%module "*modstr) -end +remote_module!(conn::Connection, modstr::String) = remotecmd(conn, Base.stdout, "%module "*modstr) +remote_module!(modstr::String) = remotecmd(_repl_client_connection, Base.stdout, "%module "*modstr) @@ -492,7 +490,7 @@ function connect_repl(host=Sockets.localhost, port::Integer=DEFAULT_PORT; conn = connect_remote(host, port; tunnel, ssh_opts, region, namespace, session_id) out_stream = stdout - prompt = ReplMaker.initrepl(c->run_remote_repl_command(conn, out_stream, c), + prompt = ReplMaker.initrepl(c->remotecmd(conn, out_stream, c), repl = Base.active_repl, valid_input_checker = valid_input_checker, prompt_text = ()->repl_prompt_text(conn), @@ -607,7 +605,7 @@ function remote_eval(host, port::Integer, cmdstr::AbstractString; local result try setup_connection!(conn) - result = run_remote_repl_command(conn, IOBuffer(), cmdstr) + result = remotecmd(conn, IOBuffer(), cmdstr) finally close(conn) end diff --git a/src/server.jl b/src/server.jl index bef1ac9..e33ec4e 100644 --- a/src/server.jl +++ b/src/server.jl @@ -252,16 +252,15 @@ function serve_repl(server::Base.IOServer; on_client_connect=nothing) while isopen(server) socket = accept(server) - session, session_id, socketidx = lock(session_lock) do - # expect session id - session_id = deserialize(socket) - session = if haskey(open_sessions, session_id) + # expect session id + session_id = deserialize(socket) + session = lock(session_lock) do + if haskey(open_sessions, session_id) push!(open_sessions[session_id].sockets, socket) open_sessions[session_id] else open_sessions[session_id] = ServerSideSession([socket], Dict(), Main) end - session, session_id, length(session.sockets) end peer = getpeername(socket) diff --git a/test/runtests.jl b/test/runtests.jl index 6e1579e..53e197d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -64,12 +64,13 @@ end @test repl_prompt_text(fake_conn("ABC", DEFAULT_PORT, is_open=false)) == "julia@ABC [disconnected]> " end -function wait_conn(host, port, use_ssh; max_tries=4) +function wait_conn(host, port, use_ssh; max_tries=4, session_id=nothing) for i=1:max_tries try return RemoteREPL.Connection(host=host, port=port, - tunnel=use_ssh ? :ssh : :none, - ssh_opts=`-o StrictHostKeyChecking=no`) + tunnel=use_ssh ? :ssh : :none, + ssh_opts=`-o StrictHostKeyChecking=no`, + session_id=session_id) catch exc if i == max_tries rethrow() @@ -81,7 +82,7 @@ function wait_conn(host, port, use_ssh; max_tries=4) end function runcommand_unwrap(conn, cmdstr) - result = RemoteREPL.run_remote_repl_command(conn, IOBuffer(), cmdstr) + result = remotecmd(conn, IOBuffer(), cmdstr) # Unwrap Text for testing purposes return result isa Text ? result.content : result end @@ -113,7 +114,7 @@ end # Use non-default port to avoid clashes with concurrent interactive use or testing. test_port = RemoteREPL.find_free_port(Sockets.localhost) -server_proc = run(`$(Base.julia_cmd()) -e "using Sockets; using RemoteREPL; serve_repl($test_port)"`, wait=false) +server_proc = run(`$(Base.julia_cmd()) --project -e "using TestEnv; TestEnv.activate(); using RemoteREPL, Sockets, UUIDs ; serve_repl($test_port)"`, wait=false) try @@ -301,7 +302,7 @@ end test_port = RemoteREPL.find_free_port(Sockets.localhost) -server_proc = run(```$(Base.julia_cmd()) -e "using Sockets; using RemoteREPL; module EvalInMod ; end; +server_proc = run(```$(Base.julia_cmd()) --project -e "using TestEnv; TestEnv.activate(); using RemoteREPL, Sockets, UUIDs ; module EvalInMod ; end; serve_repl($test_port, on_client_connect=sess->sess.in_module=EvalInMod)"```, wait=false) try @@ -311,6 +312,17 @@ try runcommand(cmdstr) = runcommand_unwrap(conn, cmdstr) @test runcommand("@__MODULE__") == "Main.EvalInMod" + + # common sessions + sid = uuid4() + conn2 = wait_conn(test_interface, test_port, use_ssh; session_id = sid) + conn3 = wait_conn(test_interface, test_port, use_ssh; session_id = sid) + + # change module to Main once + remote_module!(conn2, "Main") + + @test runcommand_unwrap(conn2, "@__MODULE__") == "Main" + @test runcommand_unwrap(conn3, "@__MODULE__") == "Main" end finally From 45870b5374c16353bb066835da0db2b070021cd2 Mon Sep 17 00:00:00 2001 From: fchrstou Date: Tue, 23 Jul 2024 16:06:48 +0200 Subject: [PATCH 3/4] Removed TestEnv --- .github/workflows/CI.yml | 1 - test/runtests.jl | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2f6d59d..0c3ee10 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -73,7 +73,6 @@ jobs: - uses: julia-actions/julia-runtest@v1 env: SSH_AUTH_SOCK: /tmp/ssh_agent.sock - - run: julia -e "import Pkg; Pkg.add(\"TestEnv\")" docs: name: Documentation runs-on: ubuntu-latest diff --git a/test/runtests.jl b/test/runtests.jl index 53e197d..746db3b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -114,7 +114,7 @@ end # Use non-default port to avoid clashes with concurrent interactive use or testing. test_port = RemoteREPL.find_free_port(Sockets.localhost) -server_proc = run(`$(Base.julia_cmd()) --project -e "using TestEnv; TestEnv.activate(); using RemoteREPL, Sockets, UUIDs ; serve_repl($test_port)"`, wait=false) +server_proc = run(`$(Base.julia_cmd()) --project -e "using RemoteREPL, Sockets, UUIDs ; serve_repl($test_port)"`, wait=false) try @@ -302,7 +302,7 @@ end test_port = RemoteREPL.find_free_port(Sockets.localhost) -server_proc = run(```$(Base.julia_cmd()) --project -e "using TestEnv; TestEnv.activate(); using RemoteREPL, Sockets, UUIDs ; module EvalInMod ; end; +server_proc = run(```$(Base.julia_cmd()) --project -e "using RemoteREPL, Sockets, UUIDs ; module EvalInMod ; end; serve_repl($test_port, on_client_connect=sess->sess.in_module=EvalInMod)"```, wait=false) try @@ -328,3 +328,5 @@ end finally kill(server_proc) end + +nothing From 90f115c1cf431a338eea899b6f71d94bf3249208 Mon Sep 17 00:00:00 2001 From: fchrstou Date: Fri, 26 Jul 2024 14:48:44 +0200 Subject: [PATCH 4/4] Check protocol first --- src/RemoteREPL.jl | 2 +- src/client.jl | 4 ++-- src/server.jl | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/RemoteREPL.jl b/src/RemoteREPL.jl index eb2f267..85d5271 100644 --- a/src/RemoteREPL.jl +++ b/src/RemoteREPL.jl @@ -9,7 +9,7 @@ export connect_repl, serve_repl, @remote, connect_remote, remotecmd, remote_modu const DEFAULT_PORT = 27754 const PROTOCOL_MAGIC = "RemoteREPL" -const PROTOCOL_VERSION = UInt32(1) +const PROTOCOL_VERSION = UInt32(2) const STDOUT_PLACEHOLDER = Symbol("#RemoteREPL_STDOUT_PLACEHOLDER") diff --git a/src/client.jl b/src/client.jl index c9ba61a..80b0fcf 100644 --- a/src/client.jl +++ b/src/client.jl @@ -139,10 +139,10 @@ function setup_connection!(conn::Connection) namespace=conn.namespace) end Sockets.nagle(socket, false) # Disables nagles algorithm. Appropriate for interactive connections. - # transmit session id - serialize(socket, conn.session_id) try verify_header(socket) + # transmit session id + serialize(socket, conn.session_id) catch exc close(socket) rethrow() diff --git a/src/server.jl b/src/server.jl index e33ec4e..913be96 100644 --- a/src/server.jl +++ b/src/server.jl @@ -177,7 +177,6 @@ end # Serve a remote REPL session to a single client function serve_repl_session(session, socket) - send_header(socket) @sync begin request_chan = Channel(1) response_chan = Channel(1) @@ -252,6 +251,8 @@ function serve_repl(server::Base.IOServer; on_client_connect=nothing) while isopen(server) socket = accept(server) + # send magic + send_header(socket) # expect session id session_id = deserialize(socket) session = lock(session_lock) do