Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,12 @@ option(
ON
)

option(
BEMAN_EXECUTION_WITH_GCD_SUPPORT
"Enable support for libdispatch (Grand Central Dispatch). Default: OFF. Values: { ON, OFF }."
OFF
)

add_subdirectory(src/beman/execution)

if(NOT BEMAN_USE_MODULES)
Expand Down
1 change: 1 addition & 0 deletions examples/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ foreach(EXAMPLE ${EXAMPLES})
set(EXAMPLE_TARGET ${PROJECT_NAME}.${EXAMPLE})
add_executable(${EXAMPLE_TARGET})
target_sources(${EXAMPLE_TARGET} PRIVATE ${EXAMPLE}.cpp)
target_compile_definitions(${EXAMPLE_TARGET} PRIVATE NOMINMAX)
if(BEMAN_USE_MODULES)
target_compile_definitions(${EXAMPLE_TARGET} PUBLIC BEMAN_HAS_MODULES)
target_link_libraries(${EXAMPLE_TARGET} PRIVATE beman::execution)
Expand Down
198 changes: 144 additions & 54 deletions include/beman/execution/detail/thread_pool_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@

#include <cassert>
#include <beman/execution/detail/common.hpp>
#if BEMAN_EXECUTION_WITH_GCD_SUPPORT && __has_include(<dispatch/dispatch.h>)
#include <dispatch/dispatch.h>
#endif
#if defined(_WIN32) && __has_include(<windows.h>)
#include <windows.h>
#include <threadpoolapiset.h>
#endif
#ifdef BEMAN_HAS_IMPORT_STD
import std;
#else
Expand All @@ -18,6 +25,7 @@ import std;
#include <mutex>
#include <new>
#include <span>
#include <system_error>
#include <thread>
#include <utility>
#endif
Expand Down Expand Up @@ -48,7 +56,7 @@ class thread_pool_backend_base

auto operator=(task_base&&) -> task_base& = delete;

virtual auto exec() noexcept -> void = 0;
virtual auto exec(int ec = 0) noexcept -> void = 0;

task_base* next;
};
Expand All @@ -57,10 +65,14 @@ class thread_pool_backend_base
explicit schedule_task(::beman::execution::parallel_scheduler_replacement::receiver_proxy& p) noexcept
: proxy(p) {}

auto exec() noexcept -> void override {
auto exec(int ec) noexcept -> void override {
auto& proxy_ref = proxy;
::std::destroy_at(this);
proxy_ref.set_value();
if (ec) [[unlikely]] {
proxy_ref.set_error(::std::make_exception_ptr(::std::system_error(ec, ::std::system_category())));
} else {
proxy_ref.set_value();
}
}

::beman::execution::parallel_scheduler_replacement::receiver_proxy& proxy;
Expand All @@ -71,18 +83,32 @@ class thread_pool_backend_base

struct single_bulk_task : task_base {
single_bulk_task(::beman::execution::parallel_scheduler_replacement::bulk_item_receiver_proxy& p,
::std::size_t shape) noexcept
: proxy(p), shape(shape) {}

auto exec() noexcept -> void override {
proxy.execute(0uz, shape);
::std::size_t shape,
bool one_by_one) noexcept
: proxy(p), shape(shape), one_by_one(one_by_one) {}

auto exec(int ec) noexcept -> void override {
if (ec == 0) [[likely]] {
if (one_by_one) {
for (::std::size_t i = 0; i < shape; ++i) {
proxy.execute(i, i + 1uz);
}
} else {
proxy.execute(0uz, shape);
}
}
auto& proxy_ref = proxy;
::std::destroy_at(this);
proxy_ref.set_value();
if (ec) [[unlikely]] {
proxy_ref.set_error(::std::make_exception_ptr(::std::system_error(ec, ::std::system_category())));
} else {
proxy_ref.set_value();
}
}

::beman::execution::parallel_scheduler_replacement::bulk_item_receiver_proxy& proxy;
::std::size_t shape;
bool one_by_one;
};

// `single_bulk_task` is small enough to fit directly into the pre-allocated storage provided by
Expand All @@ -97,6 +123,7 @@ class thread_pool_backend_base
batched_bulk_task* head;
::std::size_t chunk_count;
bool one_by_one;
int ec = 0;
::std::size_t ref_count;
};

Expand All @@ -106,25 +133,36 @@ class thread_pool_backend_base
::std::size_t j) noexcept
: cookie(cookie), proxy(proxy), i(i), j(j) {}

auto exec() noexcept -> void override {
if (cookie->one_by_one) {
for (::std::size_t k = i; k < j; ++k) {
proxy.execute(k, k + 1uz);
auto exec(int ec) noexcept -> void override {
if (ec) [[unlikely]] {
if (next == nullptr) {
cookie->ec = ec;
}
} else {
proxy.execute(i, j);
if (cookie->one_by_one) {
for (::std::size_t k = i; k < j; ++k) {
proxy.execute(k, k + 1uz);
}
} else {
proxy.execute(i, j);
}
}

::std::atomic_ref<::std::size_t> ref_count{cookie->ref_count};
if (ref_count.fetch_sub(1uz, ::std::memory_order_acq_rel) == 1uz) {
auto head = cookie->head;
const auto chunk_count = cookie->chunk_count;
auto error = ::std::error_code{cookie->ec, ::std::system_category()};
auto& proxy_ref = proxy;
::std::destroy_at(cookie);
::std::destroy_n(head, chunk_count);
::operator delete(
head, chunk_count * sizeof(batched_bulk_task), ::std::align_val_t{alignof(batched_bulk_task)});
proxy_ref.set_value();
if (error) [[unlikely]] {
proxy_ref.set_error(::std::make_exception_ptr(::std::system_error{error}));
} else {
proxy_ref.set_value();
}
}
}

Expand Down Expand Up @@ -152,16 +190,9 @@ class thread_pool_backend_base

auto operator=(thread_pool_backend_base&&) -> thread_pool_backend_base& = delete;

auto shutdown() noexcept -> void {
::std::unique_lock guard{mtx};
shutdown_requested = true;
guard.unlock();
cv.notify_all();
}

auto schedule(::beman::execution::parallel_scheduler_replacement::receiver_proxy& proxy,
::std::span<::std::byte> storage) noexcept -> void override {
push_back(::std::construct_at(reinterpret_cast<schedule_task*>(storage.data()), proxy)); // nothrow!
submit(::std::construct_at(reinterpret_cast<schedule_task*>(storage.data()), proxy)); // nothrow!
}

auto schedule_bulk_chunked(::std::size_t shape,
Expand All @@ -178,10 +209,11 @@ class thread_pool_backend_base

protected:
[[nodiscard]] static auto num_threads() noexcept -> ::std::size_t {
static const ::std::size_t count = ::std::max(1u, ::std::thread::hardware_concurrency());
static const ::std::size_t count = (::std::max)(1u, ::std::thread::hardware_concurrency());
return count;
}

private:
auto schedule_bulk(::std::size_t shape,
bool one_by_one,
::beman::execution::parallel_scheduler_replacement::bulk_item_receiver_proxy& proxy,
Expand All @@ -191,11 +223,13 @@ class thread_pool_backend_base
return;
}

const ::std::size_t chunk_length = (shape + num_threads() - 1uz) / num_threads();
const ::std::size_t chunk_count = (shape + chunk_length - 1uz) / chunk_length;
const ::std::size_t threads = num_threads();
const ::std::size_t chunk_length = shape / threads + (shape % threads != 0uz);
const ::std::size_t chunk_count = shape / chunk_length + (shape % chunk_length != 0uz);
try {
if (chunk_count == 1uz) {
push_back(::std::construct_at(reinterpret_cast<single_bulk_task*>(storage.data()), proxy, shape));
submit(::std::construct_at(
reinterpret_cast<single_bulk_task*>(storage.data()), proxy, shape, one_by_one));
} else {
auto head = static_cast<batched_bulk_task*>(::operator new(
chunk_count * sizeof(batched_bulk_task), ::std::align_val_t{alignof(batched_bulk_task)}));
Expand All @@ -206,22 +240,51 @@ class thread_pool_backend_base
batched_bulk_task* prev = nullptr;
for (::std::size_t i = 0; i < chunk_count; ++i) {
const ::std::size_t begin = i * chunk_length;
const ::std::size_t end = ::std::min(begin + chunk_length, shape);
const ::std::size_t end = begin + (::std::min)(chunk_length, shape - begin);
auto task = ::std::construct_at(head + i, cookie, proxy, begin, end);
if (prev) {
prev->next = task;
}
prev = task;
}
// NOLINTEND(*-reinterpret-cast, *-pointer-arithmetic-on-polymorphic-object, *-ctr56-cpp)
push_back(head, chunk_count);
submit(head, chunk_count);
}
} catch (...) {
proxy.set_error(::std::current_exception());
}
}

auto push_back(task_base* t, ::std::size_t n = 1uz) noexcept -> void {
virtual auto submit(task_base* t, ::std::size_t n = 1uz) noexcept -> void = 0;
};

struct simple_thread_pool_backend : ::beman::execution::detail::thread_pool_backend_base {
explicit simple_thread_pool_backend(::std::in_place_t) : workers(::new ::std::thread[num_threads()]) {}

simple_thread_pool_backend() : simple_thread_pool_backend(::std::in_place) {
for (auto& worker : ::std::span(workers.get(), num_threads())) {
worker = ::std::thread(&simple_thread_pool_backend::run, this);
}
}

~simple_thread_pool_backend() override {
shutdown();
for (auto& worker : ::std::span(workers.get(), num_threads())) {
if (worker.joinable()) {
worker.join();
}
}
}

private:
auto shutdown() noexcept -> void {
::std::unique_lock guard{mtx};
shutdown_requested = true;
guard.unlock();
cv.notify_all();
}

auto submit(task_base* t, ::std::size_t n) noexcept -> void override {
::std::unique_lock guard{mtx};
for (::std::size_t i = 0; i < n; ++i) {
if (auto prev_back = ::std::exchange(back, t)) {
Expand Down Expand Up @@ -249,41 +312,68 @@ class thread_pool_backend_base
return front ? ::std::exchange(front, front->next) : nullptr;
}

protected:
bool shutdown_requested = false;
::std::mutex mtx;
::std::condition_variable cv;
task_base* front = nullptr;
task_base* back = nullptr;
};

struct thread_pool_backend : ::beman::execution::detail::thread_pool_backend_base {
explicit thread_pool_backend(::std::in_place_t) : workers(::new ::std::thread[num_threads()]) {}

thread_pool_backend() : thread_pool_backend(::std::in_place) {
for (auto& worker : ::std::span(workers.get(), num_threads())) {
worker = ::std::thread(&thread_pool_backend::run, this);
auto run() noexcept -> void {
while (auto task = pop_front()) {
task->exec();
}
}

~thread_pool_backend() override {
shutdown();
for (auto& worker : ::std::span(workers.get(), num_threads())) {
if (worker.joinable()) {
worker.join();
}
bool shutdown_requested = false;
::std::mutex mtx;
::std::condition_variable cv;
task_base* front = nullptr;
task_base* back = nullptr;
::std::unique_ptr<::std::thread[]> workers;
};

#if BEMAN_EXECUTION_WITH_GCD_SUPPORT && __has_include(<dispatch/dispatch.h>)
struct gcd_thread_pool_backend : ::beman::execution::detail::thread_pool_backend_base {
private:
auto submit(task_base* t, ::std::size_t n) noexcept -> void override {
auto queue = ::dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
for (::std::size_t i = 0; i < n; ++i) {
auto* next = t->next;
::dispatch_async_f(queue, t, &run);
t = next;
}
assert(t == nullptr);
}

static auto run(void* ctx) noexcept -> void { static_cast<task_base*>(ctx)->exec(); }
};
#endif

#if defined(_WIN32) && __has_include(<windows.h>)
struct win_thread_pool_backend : ::beman::execution::detail::thread_pool_backend_base {
private:
auto run() noexcept -> void {
while (auto task = pop_front()) {
task->exec();
auto submit(task_base* t, ::std::size_t n) noexcept -> void override {
int ec = 0;
for (::std::size_t i = 0; i < n; ++i) {
auto* next = t->next;
if (ec) [[unlikely]] {
t->exec(ec);
} else if (!::TrySubmitThreadpoolCallback(&run, t, nullptr)) {
ec = static_cast<int>(::GetLastError());
t->exec(ec);
}
t = next;
}
assert(t == nullptr);
}

::std::unique_ptr<::std::thread[]> workers;
static auto CALLBACK run(::PTP_CALLBACK_INSTANCE, void* ctx) noexcept -> void {
static_cast<task_base*>(ctx)->exec();
}
};
#endif

#if BEMAN_EXECUTION_WITH_GCD_SUPPORT && __has_include(<dispatch/dispatch.h>)
using thread_pool_backend = gcd_thread_pool_backend;
#elif defined(_WIN32) && __has_include(<windows.h>)
using thread_pool_backend = win_thread_pool_backend;
#else
using thread_pool_backend = simple_thread_pool_backend;
#endif

} // namespace beman::execution::detail

Expand Down
1 change: 1 addition & 0 deletions src/beman/execution/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,7 @@ target_compile_definitions(
${BEMAN_EXECUTION_TARGET_NAME}
INTERFACE
BEMAN_EXECUTION_WITH_DEFAULT_PARALLEL_SCHEDULER_BACKEND=$<BOOL:${BEMAN_EXECUTION_WITH_DEFAULT_PARALLEL_SCHEDULER_BACKEND}>
BEMAN_EXECUTION_WITH_GCD_SUPPORT=$<BOOL:${BEMAN_EXECUTION_WITH_GCD_SUPPORT}>
)

if(BEMAN_USE_MODULES)
Expand Down
1 change: 1 addition & 0 deletions tests/beman/execution/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ foreach(test ${execution_tests})
else()
target_link_libraries(${TEST_EXE} PRIVATE beman::execution_headers)
endif()
target_compile_definitions(${TEST_EXE} PRIVATE NOMINMAX)
add_test(NAME ${TEST_EXE} COMMAND $<TARGET_FILE:${TEST_EXE}>)
endforeach()

Expand Down
Loading
Loading