Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 31 additions & 30 deletions include/nvexec/multi_gpu_context.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ namespace nv::execution
{
namespace _strm
{
struct multi_gpu_stream_scheduler : private stream_scheduler_env<multi_gpu_stream_scheduler>
struct multi_gpu_stream_scheduler
: private stream_scheduler_attrs<multi_gpu_stream_scheduler, set_error_t>
{
using scheduler_concept = STDEXEC::scheduler_tag;

Expand All @@ -49,7 +50,7 @@ namespace nv::execution
return sender{num_devices_, ctx_};
}

using stream_scheduler_env::query;
using stream_scheduler_attrs::query;

private:
template <class Receiver>
Expand Down Expand Up @@ -110,12 +111,37 @@ namespace nv::execution

struct sender : stream_sender_base
{
private:
struct attrs
{
template <STDEXEC::__one_of<set_value_t, set_error_t> CompletionSignal>
[[nodiscard]]
auto query(get_completion_scheduler_t<CompletionSignal>) const noexcept
-> multi_gpu_stream_scheduler
{
return multi_gpu_stream_scheduler{num_devices_, ctx_};
}

template <STDEXEC::__one_of<set_value_t, set_error_t> CompletionSignal>
[[nodiscard]]
auto query(get_completion_domain_t<CompletionSignal>) const noexcept -> stream_domain
{
return {};
}

int num_devices_;
context ctx_;
};

attrs attrs_;

public:
using completion_signatures =
STDEXEC::completion_signatures<set_value_t(), set_error_t(cudaError_t)>;

STDEXEC_ATTRIBUTE(host, device)
explicit sender(int num_devices, context ctx) noexcept
: env_{.num_devices_ = num_devices, .ctx_ = ctx}
: attrs_{.num_devices_ = num_devices, .ctx_ = ctx}
{}

template <class Receiver>
Expand All @@ -126,42 +152,17 @@ namespace nv::execution
}

[[nodiscard]]
auto get_env() const noexcept -> decltype(auto)
auto get_env() const noexcept -> attrs const &
{
return (env_);
return attrs_;
}

private:
struct attrs
{
template <class CPO>
[[nodiscard]]
auto query(get_completion_scheduler_t<CPO>, __ignore = {}) const noexcept
-> multi_gpu_stream_scheduler
{
return multi_gpu_stream_scheduler{num_devices_, ctx_};
}

int num_devices_;
context ctx_;
};

attrs env_;
};

public:
// private: TODO
int num_devices_{};
context ctx_;
};

template <>
STDEXEC_ATTRIBUTE(nodiscard)
inline auto stream_scheduler_env<multi_gpu_stream_scheduler>::query(
get_completion_scheduler_t<set_value_t>) const noexcept -> multi_gpu_stream_scheduler
{
return STDEXEC::__c_downcast<multi_gpu_stream_scheduler>(*this);
}
} // namespace _strm

using _strm::multi_gpu_stream_scheduler;
Expand Down
29 changes: 25 additions & 4 deletions include/nvexec/stream/common.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -429,10 +429,14 @@ namespace nv::execution
using decayed_tuple_t = ::cuda::std::tuple<__decay_t<Ts>...>;

template <class Sender, class Env>
using variant_storage_t =
__for_each_completion_signature_t<__completion_signatures_of_t<Sender, Env>,
decayed_tuple_t,
__munique<__q<_nullable_variant_t>>::__f>;
using variant_storage_t = __for_each_completion_signature_t<
// The variant must always be able to hold a `cudaError_t` error, since that is how
// stream receivers report `std::exception_ptr` errors. Adding it to the completion
// signatures *before* uniquing avoids duplicate alternatives.
__concat_completion_signatures_t<__completion_signatures_of_t<Sender, Env>,
completion_signatures<set_error_t(cudaError_t)>>,
decayed_tuple_t,
__munique<__q<_nullable_variant_t>>::__f>;

inline constexpr get_stream_provider_t get_stream_provider{};

Expand Down Expand Up @@ -661,6 +665,23 @@ namespace nv::execution
template <class OuterReceiver>
struct opstate_base : stream_opstate_base
{
// Stream operation states complete their receiver from a kernel. Connecting one directly
// to the host sync_wait receiver is a mistake: it means the default sync_wait was selected
// instead of the stream scheduler's. That happens when the sender passed to sync_wait does
// not report a stream completion scheduler.
static_assert(stream_receiver<OuterReceiver>
|| !__is_instance_of<OuterReceiver, __sync_wait::__receiver>,
"A stream sender was passed to sync_wait, but the sender does not report that "
"it completes "
"on a stream scheduler, so the default sync_wait was selected. This happens, "
"for example, "
"after let_value, let_error and let_stopped, whose completion scheduler "
"depends on the "
"sender their function returns. Pipe the sender through a write_attrs adaptor "
"that "
"proclaims its completion scheduler and domain before passing it to "
"sync_wait.");

using operation_state_concept = STDEXEC::operation_state_tag;
using outer_env_t = env_of_t<OuterReceiver>;
using env_t = make_stream_env_t<outer_env_t>;
Expand Down
5 changes: 3 additions & 2 deletions include/nvexec/stream/continues_on.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -239,9 +239,10 @@ namespace nv::execution::_strm

template <__decay_copyable Self, class... Env>
static consteval auto get_completion_signatures()
-> __completion_signatures_of_t<__copy_cvref_t<Self, Sender>, Env...>
{
return {};
return exec::concat_completion_signatures(
STDEXEC::get_completion_signatures<__copy_cvref_t<Self, Sender>, Env...>(),
STDEXEC::completion_signatures<set_error_t(cudaError_t)>());
}

[[nodiscard]]
Expand Down
2 changes: 1 addition & 1 deletion include/nvexec/stream/launch.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ namespace nv::execution
{
if constexpr (__callable<Fun, cudaStream_t, Args&...>)
{
return completion_signatures<set_value_t(Args...)>();
return completion_signatures<set_value_t(Args...), set_error_t(cudaError_t)>();
}
else
{
Expand Down
44 changes: 19 additions & 25 deletions include/nvexec/stream_context.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -48,26 +48,31 @@ namespace nv::execution
{
struct stream_scheduler;

template <class StreamScheduler>
struct stream_scheduler_env
{ // NOLINT(bugprone-crtp-constructor-accessibility)
template <class StreamScheduler, class... Tags>
struct stream_scheduler_attrs
{
STDEXEC_ATTRIBUTE(nodiscard)
static auto query(get_forward_progress_guarantee_t) noexcept -> forward_progress_guarantee
{
return forward_progress_guarantee::weakly_parallel;
}

template <STDEXEC::__one_of<set_value_t, Tags...> CompletionSignal>
STDEXEC_ATTRIBUTE(nodiscard)
auto query(get_completion_scheduler_t<set_value_t>) const noexcept -> StreamScheduler;
auto query(get_completion_scheduler_t<CompletionSignal>) const noexcept -> StreamScheduler
{
return STDEXEC::__c_downcast<StreamScheduler>(*this);
}

template <STDEXEC::__one_of<set_value_t, Tags...> CompletionSignal>
STDEXEC_ATTRIBUTE(nodiscard)
constexpr auto query(get_completion_domain_t<set_value_t>) const noexcept -> stream_domain
auto query(get_completion_domain_t<CompletionSignal>) const noexcept -> stream_domain
{
return {};
}
};

struct stream_scheduler : private stream_scheduler_env<stream_scheduler>
struct stream_scheduler : private stream_scheduler_attrs<stream_scheduler>
{
using scheduler_concept = STDEXEC::scheduler_tag;

Expand All @@ -85,7 +90,7 @@ namespace nv::execution
return sender{ctx_};
}

using stream_scheduler_env::query;
using stream_scheduler_attrs::query;

private:
template <class Receiver>
Expand All @@ -103,7 +108,6 @@ namespace nv::execution
private:
friend stream_context;
cudaStream_t stream_{};
cudaError_t status_{cudaSuccess};
};

struct sender : stream_sender_base
Expand All @@ -114,55 +118,45 @@ namespace nv::execution
context ctx_;

STDEXEC_ATTRIBUTE(nodiscard)
auto query(get_completion_scheduler_t<set_value_t>, __ignore = {}) const noexcept
-> stream_scheduler
auto query(get_completion_scheduler_t<set_value_t>) const noexcept -> stream_scheduler
{
return stream_scheduler{ctx_};
}

STDEXEC_ATTRIBUTE(nodiscard)
constexpr auto query(get_completion_domain_t<set_value_t>) const noexcept -> stream_domain
auto query(get_completion_domain_t<set_value_t>) const noexcept -> stream_domain
{
return {};
}
};

attrs env_;
attrs attrs_;

public:
using completion_signatures =
STDEXEC::completion_signatures<set_value_t(), set_error_t(cudaError_t)>;
using completion_signatures = STDEXEC::completion_signatures<set_value_t()>;

STDEXEC_ATTRIBUTE(host, device)
explicit sender(context ctx) noexcept
: env_{ctx}
: attrs_{ctx}
{}

template <class Receiver>
auto connect(Receiver rcvr) const & noexcept -> opstate<Receiver>
{
return opstate<Receiver>(static_cast<Receiver&&>(rcvr), env_.ctx_);
return opstate<Receiver>(static_cast<Receiver&&>(rcvr), attrs_.ctx_);
}

STDEXEC_ATTRIBUTE(nodiscard)
auto get_env() const noexcept -> attrs const &
{
return (env_);
return attrs_;
}
};

public:
// private: TODO
context ctx_;
};

template <>
STDEXEC_ATTRIBUTE(nodiscard)
inline auto stream_scheduler_env<stream_scheduler>::query(
get_completion_scheduler_t<set_value_t>) const noexcept -> stream_scheduler
{
return STDEXEC::__c_downcast<stream_scheduler>(*this);
}
} // namespace _strm

using _strm::stream_scheduler;
Expand Down
10 changes: 8 additions & 2 deletions include/stdexec/__detail/__completion_behavior.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -134,11 +134,17 @@ namespace STDEXEC
struct __common_t
{
template <__behavior... _CSs>
requires(sizeof...(_CSs) > 0)
STDEXEC_ATTRIBUTE(nodiscard, host, device)
constexpr auto operator()(__constant_t<_CSs>... __cbs) const noexcept
{
return (__cbs | ...);
if constexpr (sizeof...(_CSs) == 0)
{
return __completion_behavior::__unknown;
}
else
{
return (__cbs | ...);
}
}
};

Expand Down
Loading
Loading