diff --git a/docs/src/support.md b/docs/src/support.md index 3fd028c..cf7d632 100644 --- a/docs/src/support.md +++ b/docs/src/support.md @@ -16,9 +16,13 @@ is not supported. Client certificates require both `sslcert` and `sslkey` and use TLS 1.2. Server-only TLS connections can negotiate TLS 1.2 or TLS 1.3. -`connect_timeout` bounds the TCP connection and TLS handshake. It does not -bound PostgreSQL authentication or query execution. Use `statement_timeout` -for query execution on a direct or session-pooled connection. +`connect_timeout` bounds each connection attempt as a whole, as in libpq: the +TCP connection, GSS or TLS negotiation, authentication, and session setup +share one deadline. A host name that resolves to several addresses is dialed +within that one deadline. An attempt that runs out of time throws +`Postgres.Error` with "timeout expired". The timeout also applies to +reconnects and to `cancel_query!`. It does not bound query execution; use +`statement_timeout` for that on a direct or session-pooled connection. Keep a manual transaction or streaming cursor on the task that created it. Do not run unrelated operations on that connection until the scope ends. Use diff --git a/src/Postgres.jl b/src/Postgres.jl index 76b2628..49b7ac3 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -66,7 +66,9 @@ Supported keyword arguments. All are also available as DSN/URI options except libpq's `options` (for example `"-c search_path=myschema"` to set the default schema). Sent in the startup packet, so it also applies after an automatic reconnect. Defaults to `PGOPTIONS` when built from a DSN. -- `connect_timeout` (seconds), `statement_timeout` (milliseconds) +- `connect_timeout` (seconds): bounds each connection attempt, from the TCP + connection through authentication and session setup, as one deadline +- `statement_timeout` (milliseconds) - `sslmode` (`"disable"`, `"prefer"` (default), `"require"`, `"verify-full"`), `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`, and `sslservername`. Only `verify-full` verifies the server's certificate; `require` encrypts diff --git a/src/api/API.jl b/src/api/API.jl index 3c0d43b..53822ea 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -666,9 +666,6 @@ function authRequest(debug, len, socket, user, ::Nothing, host::String, krbsrvna end end -connectsocket(host::AbstractString, port::Integer; connect_timeout::Union{Int, Nothing}=nothing) = - connectsocket(host, port, connect_timeout) - # `host:port` for Reseau, with IPv6 literals bracketed (`[::1]:5432`) so the # address parser doesn't reject the extra colons function hostport_address(host::AbstractString, port::Integer) @@ -678,14 +675,40 @@ function hostport_address(host::AbstractString, port::Integer) return occursin(':', h) ? string("[", h, "]:", Int(port)) : string(h, ":", Int(port)) end -function connectsocket(host::AbstractString, port::Integer, @nospecialize(connect_timeout::Union{Int, Nothing})) +# The deadline of one connection attempt, as an absolute `time_ns()` instant: +# `connect_timeout` seconds from now, or 0 (none) when it is unset, zero or +# negative, as in libpq. +function connect_deadline(connect_timeout::Union{Int, Nothing}) + (connect_timeout === nothing || connect_timeout <= 0) && return Int64(0) + return Int64(time_ns()) + min(Int64(connect_timeout), Int64(9_000_000_000)) * 1_000_000_000 +end + +# A dial, read or write cut off by the attempt's deadline. The dial reports it +# inside an OpError, and TLS reports it as the cause of a TLSError. +function is_timeout(err) + err isa Reseau.IOPoll.DeadlineExceededError && return true + err isa Reseau.HostResolvers.DialTimeoutError && return true + err isa Reseau.TLS.TLSError && return err.cause isa Reseau.IOPoll.DeadlineExceededError + err isa Reseau.HostResolvers.OpError || return false + return err.err isa Reseau.HostResolvers.DialTimeoutError || err.err isa Reseau.IOPoll.DeadlineExceededError +end + +timeout_error(address::String) = Error("timeout expired while connecting to $address") + +# Dials within the attempt's `deadline` (0 for none) and leaves it on the socket +# as the read and write deadline, so the GSS or TLS negotiation, startup and +# authentication that follow share the same window. +function connectsocket(host::AbstractString, port::Integer, deadline::Int64) address = hostport_address(host, port) - return if connect_timeout === nothing - Reseau.TCP.connect(address) - else - timeout_ns = Int64(connect_timeout) * 1_000_000_000 - Reseau.TCP.connect(address; timeout_ns) + deadline == 0 && return Reseau.TCP.connect(address) + socket = try + Reseau.TCP.connect(address; deadline_ns=deadline) + catch err + is_timeout(err) && throw(timeout_error(address)) + rethrow() end + Reseau.TCP.set_deadline!(socket, deadline) + return socket end function tlsupgrade( @@ -773,32 +796,40 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos gssencmode_str == "require" && throw(Error("GSSAPI encryption required but no credential cache")) try_gss = false end - if try_gss - socket = connectsocket(host, port, connect_timeout_v) - gss_used = true + # One deadline bounds the whole attempt, the GSS fallback's second dial + # included, as libpq's connect_timeout does for one server address. + deadline = connect_deadline(connect_timeout_v) + try + if try_gss + socket = connectsocket(host, port, deadline) + gss_used = true + try + gss = gss_encrypt(socket, style, host, krbsrvname_str, gssdelegation, debug) + gss !== nothing && return _startup!(gss, debug, user, dbname, password_v, application_name_v, options_v, statement_timeout_v, host, krbsrvname_str, gssdelegation, style) + gssencmode_str == "require" && throw(Error("server doesn't support GSSAPI encryption, but it was required")) + # 'N': nothing was consumed beyond the one byte, so continue on + # this connection with TLS or plaintext, as libpq does + gss_used = false + return _connect_tls!(socket, debug, user, dbname, password_v, application_name_v, options_v, sslmode_v, sslrootcert_v, sslcert_v, sslkey_v, sslcapath_v, sslservername_v, statement_timeout_v, host, krbsrvname_str, gssdelegation, style) + catch err + close(socket) + (gss_used && gssencmode_str == "prefer") || rethrow() + debug && @info "GSSAPI encryption attempt failed; retrying without it" exception=(err, catch_backtrace()) + end + end + socket = connectsocket(host, port, deadline) + # Any failure from here on must close the socket: nothing else holds a + # reference to it, and the transport has no finalizer, so an escaping + # exception would leak the descriptor for the life of the process — + # a pool or reconnect loop against a flapping server would hit EMFILE. try - gss = gss_encrypt(socket, style, host, krbsrvname_str, gssdelegation, debug) - gss !== nothing && return _startup!(gss, debug, user, dbname, password_v, application_name_v, options_v, statement_timeout_v, host, krbsrvname_str, gssdelegation, style) - gssencmode_str == "require" && throw(Error("server doesn't support GSSAPI encryption, but it was required")) - # 'N': nothing was consumed beyond the one byte, so continue on - # this connection with TLS or plaintext, as libpq does - gss_used = false - return _connect_tls!(socket, debug, user, dbname, password_v, application_name_v, options_v, connect_timeout_v, sslmode_v, sslrootcert_v, sslcert_v, sslkey_v, sslcapath_v, sslservername_v, statement_timeout_v, host, krbsrvname_str, gssdelegation, style) - catch err + return _connect_tls!(socket, debug, user, dbname, password_v, application_name_v, options_v, sslmode_v, sslrootcert_v, sslcert_v, sslkey_v, sslcapath_v, sslservername_v, statement_timeout_v, host, krbsrvname_str, gssdelegation, style) + catch close(socket) - (gss_used && gssencmode_str == "prefer") || rethrow() - debug && @info "GSSAPI encryption attempt failed; retrying without it" exception=(err, catch_backtrace()) + rethrow() end - end - socket = connectsocket(host, port, connect_timeout_v) - # Any failure from here on must close the socket: nothing else holds a - # reference to it, and the transport has no finalizer, so an escaping - # exception would leak the descriptor for the life of the process — - # a pool or reconnect loop against a flapping server would hit EMFILE. - try - return _connect_tls!(socket, debug, user, dbname, password_v, application_name_v, options_v, connect_timeout_v, sslmode_v, sslrootcert_v, sslcert_v, sslkey_v, sslcapath_v, sslservername_v, statement_timeout_v, host, krbsrvname_str, gssdelegation, style) - catch - close(socket) + catch err + is_timeout(err) && throw(timeout_error(hostport_address(host, port))) rethrow() end end @@ -806,11 +837,10 @@ end # SSLRequest per `sslmode`, then the startup exchange on whichever transport # resulted. Each branch calls `_startup!` on a concrete socket type, so the # calls resolve statically. -function _connect_tls!(socket::Reseau.TCP.Conn, debug::Bool, user::String, dbname::String, @nospecialize(password::Union{String, Nothing}), @nospecialize(application_name::Union{String, Nothing}), @nospecialize(options::Union{String, Nothing}), @nospecialize(connect_timeout::Union{Int, Nothing}), @nospecialize(sslmode::Union{String, Nothing}), @nospecialize(sslrootcert::Union{String, Nothing}), @nospecialize(sslcert::Union{String, Nothing}), @nospecialize(sslkey::Union{String, Nothing}), @nospecialize(sslcapath::Union{String, Nothing}), @nospecialize(sslservername::Union{String, Nothing}), @nospecialize(statement_timeout::Union{Int, Nothing}), host::String, krbsrvname::String, gssdelegation::Bool, style::AbstractPostgresStyle) +function _connect_tls!(socket::Reseau.TCP.Conn, debug::Bool, user::String, dbname::String, @nospecialize(password::Union{String, Nothing}), @nospecialize(application_name::Union{String, Nothing}), @nospecialize(options::Union{String, Nothing}), @nospecialize(sslmode::Union{String, Nothing}), @nospecialize(sslrootcert::Union{String, Nothing}), @nospecialize(sslcert::Union{String, Nothing}), @nospecialize(sslkey::Union{String, Nothing}), @nospecialize(sslcapath::Union{String, Nothing}), @nospecialize(sslservername::Union{String, Nothing}), @nospecialize(statement_timeout::Union{Int, Nothing}), host::String, krbsrvname::String, gssdelegation::Bool, style::AbstractPostgresStyle) password_v = password::Union{String, Nothing} application_name_v = application_name::Union{String, Nothing} options_v = options::Union{String, Nothing} - connect_timeout_v = connect_timeout::Union{Int, Nothing} sslmode_v = sslmode::Union{String, Nothing} sslrootcert_v = sslrootcert::Union{String, Nothing} sslcert_v = sslcert::Union{String, Nothing} @@ -825,8 +855,9 @@ function _connect_tls!(socket::Reseau.TCP.Conn, debug::Bool, user::String, dbnam writemessage(socket, debug, '\0', Int32(80877103)) mt = read(socket, UInt8) if mt == UInt8('S') - # upgrade socket to tls and do handshake - tls = tlsupgrade(socket, connect_timeout_v, + # upgrade socket to tls and do handshake; the socket's connect + # deadline bounds it + tls = tlsupgrade(socket, nothing, sslservername_v isa String ? sslservername_v : host, sslmode_str == "verify-full", sslcert_v, sslkey_v, sslrootcert_v, sslcapath_v) @@ -867,6 +898,8 @@ function _startup!(socket, debug::Bool, user::String, dbname::String, @nospecial close_and_throw(socket, Error("server did not confirm UTF8 client_encoding")) buffered = BufferedConn(socket) align_session_formats!(buffered, server_params, debug, statement_timeout_v) + # Established: lift the connect deadline so it cannot cut off a later statement. + Reseau.TCP.set_deadline!(socket isa Reseau.TCP.Conn ? socket : socket.tcp, 0) return buffered, pid, skey, server_params end @@ -1659,7 +1692,7 @@ function cancel_request(host::String, port::Int, pid::Int32, skey::Int32, debug: gssencmode_str in GSSENCMODES || throw(Error("invalid gssencmode: $gssencmode_str")) sslmode_str in SSLMODES || throw(Error("invalid sslmode: $sslmode_str")) krbsrvname_str = krbsrvname === nothing ? "postgres" : krbsrvname::String - socket = connectsocket(host, port, connect_timeout_v) + socket = connectsocket(host, port, connect_deadline(connect_timeout_v)) stream = socket refused = nothing sent = false @@ -1677,7 +1710,7 @@ function cancel_request(host::String, port::Int, pid::Int32, skey::Int32, debug: writemessage(socket, debug, '\0', Int32(80877103)) mt = read(socket, UInt8) if mt == UInt8('S') - stream = tlsupgrade(socket, connect_timeout_v, + stream = tlsupgrade(socket, nothing, sslservername isa String ? sslservername::String : host, sslmode_str == "verify-full", sslcert::Union{String, Nothing}, sslkey::Union{String, Nothing}, diff --git a/src/connection_string.jl b/src/connection_string.jl index db27db2..df85c2f 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -160,8 +160,9 @@ const IGNORED_PARAMS = Set([ # valid UTF-8 either, but lowercase accepts them, so they parse as before. has_malformed_char(s::String) = any(c -> Base.ismalformed(c) || Base.isoverlong(c), s) -# sslmode and gssencmode are case-insensitive, as in libpq. A malformed value -# can never match and cannot be lowercased. +# Postgres.jl compares sslmode and gssencmode case-insensitively; libpq +# compares them exactly. A malformed value can never match and cannot be +# lowercased. function check_mode(key::String, value::Union{String, Nothing}, accepted::Tuple{Vararg{String}}) value === nothing && return nothing !has_malformed_char(value) && lowercase(value) in accepted && return nothing diff --git a/test/connect_timeout_fake_server.jl b/test/connect_timeout_fake_server.jl new file mode 100644 index 0000000..fb4a844 --- /dev/null +++ b/test/connect_timeout_fake_server.jl @@ -0,0 +1,101 @@ +# connect_timeout against scripted servers that stall at each stage of a +# connection attempt. Reuses the fake-server helpers from gssapi.jl, +# isvalid_fake_server.jl, and execute_fake_server.jl. + +timeout_connection(port; kw...) = Postgres.Connection(; host="127.0.0.1", port=port, user="u", dbname="db", + gssencmode="disable", connect_timeout=1, kw...) + +function expect_connect_timeout(f) + started = time() + err = connect_err(f) + @test err isa Postgres.Error + @test occursin("timeout expired", errtext(err)) + @test time() - started < 5 + return +end + +# answers one statement's Parse/Describe/Bind/Execute/Sync with no data +function serve_statement(sock) + read_unnamed_segment(sock) + send(sock, no_data_replies("DO")) + return +end + +function test_connect_timeout_fake_server() +@testset "connect_timeout bounds the whole connection attempt" begin + @testset "server silent after accepting (sslmode=$sslmode)" for sslmode in ("disable", "prefer") + with_fake_server((sock, n) -> drain(sock)) do port, accepted + expect_connect_timeout(() -> timeout_connection(port; sslmode=sslmode)) + end + end + + @testset "TLS handshake stalls" begin + with_fake_server((sock, n) -> begin + read_request(sock) + send(sock, [UInt8('S')]) + drain(sock) + end) do port, accepted + expect_connect_timeout(() -> timeout_connection(port; sslmode="require")) + end + end + + @testset "one deadline for all stages, not one per stage" begin + # each stage answers within connect_timeout, but together they take longer + with_fake_server((sock, n) -> begin + read_request(sock) + sleep(0.6) + send(sock, [UInt8('N')]) + read_startup(sock) + sleep(0.6) + ready!(sock) + drain(sock) + end) do port, accepted + expect_connect_timeout(() -> timeout_connection(port; sslmode="prefer")) + end + end + + @testset "a statement after connect_timeout has passed succeeds" begin + with_fake_server(serve_idle(sock -> (serve_statement(sock); drain(sock)))) do port, accepted + conn = timeout_connection(port; sslmode="disable") + sleep(1.5) + @test Postgres.command_tag(DBInterface.execute(conn, "DO 1")) == "DO" + @test isopen(conn) + close(conn) + end + end + + @testset "... also after a reconnect" begin + handler = function (sock, n) + serve_idle(sock -> begin + if n == 1 + read_unnamed_segment(sock) + close(sock) + else + serve_statement(sock) + serve_statement(sock) + drain(sock) + end + end)(sock, n) + end + with_fake_server(handler) do port, accepted + conn = timeout_connection(port; sslmode="disable", reconnect=true) + @test connect_err(() -> DBInterface.execute(conn, "DO 1")) isa EOFError + @test_logs (:warn, r"reconnected") DBInterface.execute(conn, "DO 1") + @test accepted[] == 2 + sleep(1.5) + @test Postgres.command_tag(DBInterface.execute(conn, "DO 1")) == "DO" + close(conn) + end + end + + @testset "cancel_request gives up when the server stalls" begin + with_fake_server((sock, n) -> drain(sock)) do port, accepted + started = time() + # the SSLRequest is never answered + @test !Postgres.API.cancel_request("127.0.0.1", port, Int32(1), Int32(1), false, "prefer", + nothing, nothing, nothing, nothing, nothing, 1) + @test time() - started < 5 + end + end +end +end diff --git a/test/execute_fake_server.jl b/test/execute_fake_server.jl index 6cbb4dd..dcb5b7a 100644 --- a/test/execute_fake_server.jl +++ b/test/execute_fake_server.jl @@ -25,7 +25,7 @@ function test_execute_fake_server() with_fake_server((sock, n) -> begin send(sock, vcat(pgmsg('Z', [UInt8('I')]), pgmsg('d', big), pgmsg('Z', [UInt8('T')]))) end) do port, accepted - io = Postgres.API.BufferedConn(Postgres.API.connectsocket("127.0.0.1", port, nothing)) + io = Postgres.API.BufferedConn(Postgres.API.connectsocket("127.0.0.1", port, Int64(0))) sleep(0.2) # let the whole burst reach the socket @test Postgres.API.readheader(io) == (UInt8('Z'), 1) # one transport read fetched more than the 5-byte header diff --git a/test/gssapi.jl b/test/gssapi.jl index bfbf295..479d0e5 100644 --- a/test/gssapi.jl +++ b/test/gssapi.jl @@ -458,6 +458,23 @@ function test_gssapi_protocol() end end + @testset "connect_timeout bounds a stalled GSS handshake" begin + style = FakeGSSStyle() + handler = function (sock, n) + code, _ = read_request(sock) + code == GSSENC_REQUEST || error("expected GSSENCRequest, got $code") + send(sock, [UInt8('G')]) + readframe(sock) # the client's token; the server's never comes + drain(sock) + end + with_fake_server(handler) do port, accepted + started = time() + err = connect_err(() -> Postgres.Connection(host="127.0.0.1", port=port, user="u", gssencmode="require", connect_timeout=1, style=style)) + @test err isa Postgres.Error && occursin("timeout expired", err.message) + @test time() - started < 5 + end + end + @testset "cancel_request accepts every sslmode and gssencmode value, in any case" begin cancels = Ref(0) handler = function (sock, n) diff --git a/test/runtests.jl b/test/runtests.jl index a68f419..3bb60d2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -498,6 +498,7 @@ include("result_decoding.jl") include("gssapi.jl") include("isvalid_fake_server.jl") include("execute_fake_server.jl") +include("connect_timeout_fake_server.jl") include("execute_script_fake_server.jl") include("executemany.jl") include("notification_deadlines.jl") @@ -1061,6 +1062,7 @@ include("trim_compile_tests.jl") test_isvalid_fake_server() test_isvalid_fragmentation() test_execute_fake_server() + test_connect_timeout_fake_server() test_execute_script_fake_server() test_executemany_protocol() test_notification_deadlines()